如何在 Java 中解析大 (50 GB) XML 文件

问题描述

目前我正在尝试使用 SAX 解析器，但大约 3/4 的文件完全冻结了，我尝试分配更多内存等但没有得到任何改进.

Currently im trying to use a SAX Parser but about 3/4 through the file it just completely freezes up, i have tried allocating more memory etc but not getting any improvements.

有什么办法可以加快速度吗?更好的方法?

Is there any way to speed this up? A better method?

将其剥离，所以我现在有以下代码，当在命令行中运行时，它仍然没有我想要的那么快.

Stripped it to bare bones, so i now have the following code and when running in command line it still doesn't go as fast as i would like.

使用java -Xms-4096m -Xmx8192m -jar reader.jar"运行它，我得到超过文章 700000 附近的 GC 开销限制

Running it with "java -Xms-4096m -Xmx8192m -jar reader.jar" i get a GC overhead limit exceeded around article 700000

主要:

public class Read {
    public static void main(String[] args) {       
       pages = XMLManager.getPages();
    }
}

XML 管理器

public class XMLManager {
    public static ArrayList<Page> getPages() {

    ArrayList<Page> pages = null; 
    SAXParserFactory factory = SAXParserFactory.newInstance();

    try {

        SAXParser parser = factory.newSAXParser();
        File file = new File("..\enwiki-20140811-pages-articles.xml");
        PageHandler pageHandler = new PageHandler();

        parser.parse(file, pageHandler);
        pages = pageHandler.getPages();

    } catch (ParserConfigurationException e) {
        e.printStackTrace();
    } catch (SAXException e) {
        e.printStackTrace();
    } catch (IOException e) {
        e.printStackTrace();
    }


    return pages;
    }    
}

页面处理程序

public class PageHandler extends DefaultHandler{

    private ArrayList<Page> pages = new ArrayList<>();
    private Page page;
    private StringBuilder stringBuilder;
    private boolean idSet = false;

    public PageHandler(){
        super();
    }

    @Override
    public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {

        stringBuilder = new StringBuilder();

         if (qName.equals("page")){

            page = new Page();
            idSet = false;

        } else if (qName.equals("redirect")){
             if (page != null){
                 page.setRedirecting(true);
             }
        }
    }

     @Override
     public void endElement(String uri, String localName, String qName) throws SAXException {

         if (page != null && !page.isRedirecting()){

             if (qName.equals("title")){

                 page.setTitle(stringBuilder.toString());

             } else if (qName.equals("id")){

                 if (!idSet){

                     page.setId(Integer.parseInt(stringBuilder.toString()));
                     idSet = true;

                 }

             } else if (qName.equals("text")){

                 String articleText = stringBuilder.toString();

                 articleText = articleText.replaceAll("(?s)<ref(.+?)</ref>", " "); //remove references
                 articleText = articleText.replaceAll("(?s)\{\{(.+?)\}\}", " "); //remove links underneath headings
                 articleText = articleText.replaceAll("(?s)==See also==.+", " "); //remove everything after see also
                 articleText = articleText.replaceAll("\|", " "); //Separate multiple links
                 articleText = articleText.replaceAll("\n", " "); //remove new lines
                 articleText = articleText.replaceAll("[^a-zA-Z0-9- \s]", " "); //remove all non alphanumeric except dashes and spaces
                 articleText = articleText.trim().replaceAll(" +", " "); //convert all multiple spaces to 1 space

                 Pattern pattern = Pattern.compile("([\S]+\s*){1,75}"); //get first 75 words of text
                 Matcher matcher = pattern.matcher(articleText);
                 matcher.find();

                 try {
                     page.setSummaryText(matcher.group());
                 } catch (IllegalStateException se){
                     page.setSummaryText("None");
                 }
                 page.setText(articleText);

             } else if (qName.equals("page")){

                 pages.add(page);
                 page = null;

            }
        } else {
            page = null;
        }
     }

     @Override
     public void characters(char[] ch, int start, int length) throws SAXException {
         stringBuilder.append(ch,start, length); 
     }

     public ArrayList<Page> getPages() {
         return pages;
     }
}

推荐答案

您的解析代码可能工作正常，但是您正在加载的数据量可能太大而无法在 ArrayList.

public interface PageProcessor {
    void process(Page page);
}

public class Read  {
    public static void main(String[] args) {

        XMLManager.load(new PageProcessor() {
            @Override
            public void process(Page page) {
                // Obviously you want to do something other than just printing, 
                // but I don't know what that is...
                System.out.println(page);
           }
        }) ;
    }

}


public class XMLManager {

    public static void load(PageProcessor processor) {
        SAXParserFactory factory = SAXParserFactory.newInstance();

        try {

            SAXParser parser = factory.newSAXParser();
            File file = new File("pages-articles.xml");
            PageHandler pageHandler = new PageHandler(processor);

            parser.parse(file, pageHandler);

        } catch (ParserConfigurationException e) {
            e.printStackTrace();
        } catch (SAXException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }

    }
}

public class PageHandler extends DefaultHandler {

    private final PageProcessor processor;
    private Page page;
    private StringBuilder stringBuilder;
    private boolean idSet = false;

    public PageHandler(PageProcessor processor) {
        this.processor = processor;
    }

    @Override
    public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {
         //Unchanged from your implementation
    }

    @Override
    public void characters(char[] ch, int start, int length) throws SAXException {
         //Unchanged from your implementation
    }

    @Override
    public void endElement(String uri, String localName, String qName) throws SAXException {
            //  Elide code not needing change

            } else if (qName.equals("page")){

                processor.process(page);
                page = null;

            }
        } else {
            page = null;
        }
    }

}