diff --git a/README.md b/README.md index 071f7d2..7e6923d 100644 --- a/README.md +++ b/README.md @@ -49,10 +49,27 @@ conf.setNaaccrVersion("220"); new MetafileTranslator().executeFullTranslation(conf); ``` +## Command Line +The compiled JAR can be executed from the command line to perform a full translation. + +**Command Line Options:** +- `-p`: The working folder containing the metafile +- `-c`: The file name of the configuration properties file within the working folder + +**Example usage:** +```sh +$ java -jar validation-translation.jar -c config.properties -p WORKING_DIR +``` +For an example of a translation configuration, see `examples/config.properties` + +## Pre-Compiled Groovy Edits +For faster loading of large metafiles, the optionally generated Groovy source code can be pre-compiled into bytecode. The folder `examples/compile-groovy` +contains an example of a build.gradle and settings.gradle that can be placed in the output folder to generate a JAR containing compiled bytecode. + ## About SEER This library was developed through the [SEER](http://seer.cancer.gov/) program. The Surveillance, Epidemiology and End Results program is a premier source for cancer statistics in the United States. The SEER program collects information on incidence, prevalence and survival from specific geographic areas representing -a large portion of the US population and reports on all these data plus cancer mortality data for the entire country. \ No newline at end of file +a large portion of the US population and reports on all these data plus cancer mortality data for the entire country. diff --git a/build.gradle b/build.gradle index f71e546..fc4cb6e 100644 --- a/build.gradle +++ b/build.gradle @@ -31,10 +31,12 @@ dependencies { implementation 'net.sf.squirrel-sql.thirdparty-non-maven:java-cup:0.11a' implementation 'org.xerial:sqlite-jdbc:3.43.2.1' implementation 'org.apache.logging.log4j:log4j-api:2.21.1' + implementation 'commons-cli:commons-cli:1.6.0' + implementation 'org.apache.logging.log4j:log4j-core:2.21.1' + implementation 'org.slf4j:slf4j-nop:2.0.12' testImplementation 'junit:junit:4.13.2' testImplementation 'com.imsweb:data-generator:1.31' - testImplementation 'org.apache.logging.log4j:log4j-core:2.21.1' } // enforce UTF-8, display the compilation warnings @@ -67,7 +69,7 @@ jar { 'Built-By': System.getProperty('user.name'), 'Built-Date': new Date(), 'Built-JDK': System.getProperty('java.version'), - 'Automatic-Module-Name': 'com.imsweb.validation-translation' + 'Automatic-Module-Name': 'com.imsweb.validation-translation', ) } } @@ -140,11 +142,34 @@ tasks.register('generateRegexParser') { } } +// use this task to generate a fat JAR to easily run a CLI +tasks.register('buildFatJar',Jar) { + manifest { + attributes('Implementation-Title': project.name, + 'Implementation-Version': project.version, + 'Implementation-Vendor': 'Information Management Services Inc.', + 'Created-By': System.properties.getProperty('java.vm.version') + ' (' + System.properties.getProperty('java.vm.vendor') + ')', + 'Built-By': System.getProperty('user.name'), + 'Built-Date': new Date(), + 'Built-JDK': System.getProperty('java.version'), + 'Automatic-Module-Name': 'com.imsweb.validation-translation', + 'Main-Class': 'com.imsweb.validation.translation.MetafileTranslatorCli' + ) + } + + from { + configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) } + } + duplicatesStrategy=DuplicatesStrategy.EXCLUDE + with jar +} + // Nexus vulnerability scan (https://github.com/sonatype-nexus-community/scan-gradle-plugin) ossIndexAudit { outputFormat = 'DEPENDENCY_GRAPH' printBanner = false + excludeVulnerabilityIds = [ 'CVE-2022-42003', 'CVE-2022-42004', diff --git a/examples/config.properties b/examples/config.properties new file mode 100755 index 0000000..ace1333 --- /dev/null +++ b/examples/config.properties @@ -0,0 +1,16 @@ +# metafile filename. should be located in the working path along with this configuration file +metafile-name=testing-metafile-v2.smf +# prefix for the validator rule IDs to generate +translation-prefix=TEST +# working directory for a previous translation run (optional). this will be relative to the path where the program is executed +previous-working-directory-path=testing-metafile-v1 +# base naaccr dictionary version to use to resolve fields +naaccr-version=230 +# user dictionary for resolving fields should the base naaccr dictionary fail to resolve a field +user-dictionary-file=test-user-dictionary.xml +# field mapping csv file for resolving fields should the base and user naaccr dictionaries fail to resolve a field +field-mappings-file=test-field-mappings.csv +# generate groovy source file? 1, yes, or true will enable groovy source generation +generate-groovy-src=1 +# number of files to split the groovy source into. this can help if groovy compilation runs out of memory. +groovy-src-num-files=2 diff --git a/examples/test-field-mappings.csv b/examples/test-field-mappings.csv new file mode 100644 index 0000000..9a32ece --- /dev/null +++ b/examples/test-field-mappings.csv @@ -0,0 +1,5 @@ +1234,xmlFieldNameMappingForNaaccrId +9101,stateSpecificField1 +9102,stateSpecificField2 +10010,userDictionaryField1 +10020,userDictionaryField2 diff --git a/gradlew b/gradlew old mode 100644 new mode 100755 diff --git a/src/main/java/com/imsweb/validation/translation/CsvFieldResolver.java b/src/main/java/com/imsweb/validation/translation/CsvFieldResolver.java new file mode 100755 index 0000000..d64d2e3 --- /dev/null +++ b/src/main/java/com/imsweb/validation/translation/CsvFieldResolver.java @@ -0,0 +1,84 @@ +package com.imsweb.validation.translation; + +import com.imsweb.naaccrxml.NaaccrXmlDictionaryUtils; +import com.imsweb.validation.translation.metafile.MetafileField; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.Map; + +/** + * Implementation of the FieldResolver that reads a comma-delimted file containing item number + * and field name and uses that to supplement NAACCR and user dictionary fields when resolving + * fields used in a EDITS metafile. + * (note: does not conform to RFC 4180! quoting and escaping are not supported) + */ +@SuppressWarnings("unused") +public class CsvFieldResolver extends FieldResolver { + private static final Logger _LOG = LogManager.getLogger(CsvFieldResolver.class); + + private Map _mappings; + + public CsvFieldResolver() { + setMappings(new HashMap<>()); + } + + /** + * Reads the comma delimited file into memory. If mappings are already in place, + * new mappings are loaded over top of the old ones. This way, multiple mapping files + * can be loaded. + * @param mappingsFile + * @throws IOException + */ + public void loadMappingsFile(File mappingsFile) throws IOException { + if (mappingsFile == null) { + throw new IOException("Mappings file cannot be null"); + } + if (! mappingsFile.exists()) { + throw new IOException("Mappings file does not exist"); + } + + for (String line : Files.readAllLines(mappingsFile.toPath(), StandardCharsets.UTF_8)) { + String[] mapping = line.trim().split("\\s*,\\s*"); + if (mapping.length != 2) { + throw new IOException("Invalid line in mapping file: " + line); + } + getMappings().put(Integer.parseInt(mapping[0]), mapping[1]); + _LOG.info(" >> Loaded mapping from " + mappingsFile.getName() + ": " + mapping[0] + "->" + mapping[1]); + } + } + + @Override + protected String resolveFieldPostDictionary(MetafileField field, TranslationConfiguration conf) { + String propName = super.resolveFieldPostDictionary(field, conf); + + // if unable to resolve by superclass implementation, try the in memory mapping + if (propName == null) { + int itemNumber = field.getNumber(); + if (getMappings().containsKey(itemNumber)) { + propName = getMappings().get(itemNumber); + } + } + + // if unable to resolve from the mapping file, create a (properly named) dummy item and warn the user + if (propName == null) { + propName = NaaccrXmlDictionaryUtils.createNaaccrIdFromItemName(field.getName()); + _LOG.info(" >> Unsupported field: " + field.getName() + " (#" + field.getNumber() + "); deriving the ID from the name: " + propName); + } + return propName; + } + + public Map getMappings() { + return _mappings; + } + + public void setMappings(Map mappings) { + this._mappings = mappings; + } + +} diff --git a/src/main/java/com/imsweb/validation/translation/MetafileTranslatorCli.java b/src/main/java/com/imsweb/validation/translation/MetafileTranslatorCli.java new file mode 100644 index 0000000..5bf15b7 --- /dev/null +++ b/src/main/java/com/imsweb/validation/translation/MetafileTranslatorCli.java @@ -0,0 +1,162 @@ +package com.imsweb.validation.translation; + +import com.imsweb.naaccrxml.NaaccrXmlDictionaryUtils; +import com.imsweb.naaccrxml.entity.dictionary.NaaccrDictionary; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.HelpFormatter; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +public class MetafileTranslatorCli { + private static final Logger _LOG = LogManager.getLogger(MetafileTranslatorCli.class); + + public static void main(String[] args) + throws Exception { + CommandLineParser parser = new DefaultParser(); + + + // set up options + Options opts = new Options(); + opts.addRequiredOption("p", "path", true, "Path to working directory containing metafile where output will be placed"); + opts.addRequiredOption("c", "config", true, "Filename of configuration properties file within the working directory to use for this translation"); + opts.addOption("h", "help", false, "Prints this help message"); + + try { + CommandLine cmdLine = parser.parse(opts, args); + + // if help requested, print and exit + if (cmdLine.hasOption("h")) { + HelpFormatter helpFormatter = new HelpFormatter(); + helpFormatter.printHelp(MetafileTranslatorCli.class.getName(), opts); + return; + } + + // process options into translation configuration + TranslationConfiguration conf = getConfiguration(cmdLine.getOptionValue("p"), cmdLine.getOptionValue("c")); + + // run the translation + new MetafileTranslator().executeFullTranslation(conf); + } + catch (ParseException e) { + HelpFormatter helpFormatter = new HelpFormatter(); + helpFormatter.printHelp("validation-translation", opts); + } + } + + /** + * Creates a TranslationConfiguration object from the configuration properties file + * @param path - path to the directory containing the configuration properties file + * @param filename - name of the configuration properties file + * @return + */ + private static TranslationConfiguration getConfiguration(String path, String filename) + throws IOException, TranslationException { + // make sure the directory exists + File configDir = new File(path); + if (! configDir.exists() || ! configDir.isDirectory()) { + throw new IOException("Working directory path must point to an existing directory"); + } + + // check to see if the properties file exists + File configFile = new File(configDir, filename); + if (! configFile.exists() || ! configFile.isFile()) { + throw new IOException("Configuration file not found"); + } + + // load the properties file + Properties configProperties = new Properties(); + try (InputStream inputStream = Files.newInputStream(configFile.toPath())) { + configProperties.load(inputStream); + } + + // set up the translation configuration based on contents of the config properties file + TranslationConfiguration translationConfiguration = new TranslationConfiguration(); + translationConfiguration.setWorkingDirectoryPath(path); + + // load the required items + if (configProperties.containsKey("metafile-name")) { + translationConfiguration.setMetafileName(configProperties.getProperty("metafile-name")); + } + else { + throw new TranslationException("No metafile name specified in configuration"); + } + + if (configProperties.containsKey("translation-prefix")) { + translationConfiguration.setTranslationPrefix(configProperties.getProperty("translation-prefix")); + } + else { + throw new TranslationException("Translation prefix is not specified in configuration"); + } + + if (configProperties.containsKey("naaccr-version")) { + translationConfiguration.setNaaccrVersion(configProperties.getProperty("naaccr-version")); + } + else { + throw new TranslationException("No NAACCR version specified in configuration"); + } + + // load optional previous translation output path + if (configProperties.containsKey("previous-working-directory-path")) { + File previousWorkingDirectoryPath = new File(configProperties.getProperty("previous-working-directory-path")); + if (! previousWorkingDirectoryPath.exists() || !previousWorkingDirectoryPath.isDirectory()) { + throw new IOException("Previous working directory specified in configuration file not found"); + } + translationConfiguration.setPreviousWorkingDirectoryPath(configProperties.getProperty("previous-working-directory-path")); + } + + // if a field mapping csv file is specified then create a new CsvFieldResolver to handle it + if (configProperties.containsKey("field-mappings-file")) { + File fieldMappingsFile = new File(configDir, configProperties.getProperty("field-mappings-file")); + if (! fieldMappingsFile.exists() || ! fieldMappingsFile.isFile()) { + throw new IOException("Field mappings file specified in configuration file not found"); + } + CsvFieldResolver csvFieldResolver = new CsvFieldResolver(); + csvFieldResolver.loadMappingsFile(fieldMappingsFile); + + translationConfiguration.setFieldResolver(csvFieldResolver); + } + + // if a user dictionary is specified, load it + if (configProperties.containsKey("user-dictionary-file")) { + File userDictionaryFile = new File(configDir, configProperties.getProperty("user-dictionary-file")); + if (! userDictionaryFile.exists() || ! userDictionaryFile.isFile()) { + throw new IOException("User dictionary file specified in configuration file not found"); + } + + List userDictionaries = new ArrayList(1); + NaaccrDictionary dict = NaaccrXmlDictionaryUtils.readDictionary(userDictionaryFile); + userDictionaries.add(dict); + _LOG.info(" >> Loaded user dictionary " + userDictionaryFile.getName()); + + translationConfiguration.setUserDefinedDictionaries(userDictionaries); + } + + // check to see if groovy source code needs to be generated + if (configProperties.containsKey("generate-groovy-src")) { + // if it is truthy, enable source generation + if (configProperties.getProperty("generate-groovy-src").equals("1") + || configProperties.getProperty("generate-groovy-src").equalsIgnoreCase("yes") + || configProperties.getProperty("generate-groovy-src").equalsIgnoreCase("true")) { + translationConfiguration.setGenerateGroovySourceCode(true); + } + } + // check if groovy source needs to be split + if (configProperties.containsKey("groovy-src-num-files")) { + translationConfiguration.setGroovySourceCodeNumFiles(Integer.parseInt(configProperties.getProperty("groovy-src-num-files"))); + } + + return translationConfiguration; + } +} diff --git a/src/main/resources/log4j2.properties b/src/main/resources/log4j2.properties new file mode 100755 index 0000000..61b2df3 --- /dev/null +++ b/src/main/resources/log4j2.properties @@ -0,0 +1,5 @@ +rootLogger=INFO, STDOUT +appender.console.type = Console +appender.console.name = STDOUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%-5level] %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %c{1} - %msg%n