View Javadoc

    1   /***
    2    * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
    3    */
    4   package net.sourceforge.pmd.cpd;
    5
    6   import net.sourceforge.pmd.cpd.cppast.CPPParserTokenManager;
    7   import net.sourceforge.pmd.cpd.cppast.Token;
    8   import net.sourceforge.pmd.cpd.cppast.TokenMgrError;
    9   import net.sourceforge.pmd.cpd.cppast.SimpleCharStream;
    10
    11  import java.io.StringReader;
    12  import java.util.ArrayList;
    13  import java.util.List;
    14
    15  public class CPPTokenizer implements Tokenizer {
    16      protected String EOL = System.getProperty("line.separator", "\n");
    17
    18      public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
    19          StringBuffer sb = sourceCode.getCodeBuffer();
    20          try {
    21              CPPParserTokenManager tokenManager = new CPPParserTokenManager(new SimpleCharStream(new StringReader(sb.toString())));
    22              Token currToken = tokenManager.getNextToken();
    23              while (currToken.image.length() > 0) {
    24                  tokenEntries.add(new TokenEntry(currToken.image, sourceCode.getFileName(), currToken.beginLine));
    25                  currToken = tokenManager.getNextToken();
    26              }
    27              tokenEntries.add(TokenEntry.getEOF());
    28              System.out.println("Added " + sourceCode.getFileName());
    29          } catch (TokenMgrError err) {
    30              err.printStackTrace();
    31              System.out.println("Skipping " + sourceCode.getFileName() + " due to parse error");
    32              List emptyCode = new ArrayList();
    33              emptyCode.add("");
    34              tokenEntries.add(TokenEntry.getEOF());
    35          }
    36      }
    37  }