From 85faca833724bb1b577216bdcbd3f0faceef043f Mon Sep 17 00:00:00 2001 From: Jake Huneau Date: Wed, 22 Jul 2026 18:56:25 -0400 Subject: [PATCH] Add $msg.hash filename parameter for content-based dedup of received files Adds a $msg.hash.$ parameter to the message filename templates. The algorithm is md5, sha1, sha256 or sha512, optionally followed by an underscore and a truncation length (e.g. $msg.hash.sha256_16$ for the first 16 hex characters). It hex-encodes a digest of the message payload, computed lazily only when referenced. Including it in store_received_file_to (or the global received-file template) dedups by name and content: a re-delivery of the same file resolves to the same path and overwrites, while changed content gets a new name. SHA-256 truncated to 16 chars is a good default - 64 bits is far more than enough collision resistance per filename. Adds a unit test and a commented example in partnerships.xml. --- RELEASE-NOTES.md | 1 + Server/src/config/partnerships.xml | 6 ++ .../org/openas2/params/MessageParameters.java | 61 +++++++++++++ .../params/MessageParametersHashTest.java | 85 +++++++++++++++++++ 4 files changed, 153 insertions(+) create mode 100644 Server/src/test/java/org/openas2/params/MessageParametersHashTest.java diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 32b1556e1..220a0696f 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -25,6 +25,7 @@ This is a minor enhancement and bugfix release. 9. Add DbPartnershipFactory: partnerships can optionally be stored in a database (Azure SQL, PostgreSQL, MySQL, Oracle or the embedded H2) instead of the partnerships XML file. See the commented example in config.xml. The required tables are included in db_ddl.sql and openas2-schema.xml and are only needed when using the database partnership store. 10. Add mutual TLS (client certificate) authentication for outbound HTTPS connections using the https_client_keystore, https_client_keystore_password and https_client_cert_alias partnership attributes (or properties for a global client identity). See the commented example in partnerships.xml. 11. Add JmsPollingModule: an alternative outbound intake that consumes work from an AMQP 1.0 message queue (Azure Service Bus or any AMQP broker via Apache Qpid JMS) instead of polling a directory. An external producer publishes a queue message identifying the sender/receiver AS2 IDs and the file path; the file is sent through the existing pipeline and the broker owns retry/dead-lettering. See the commented example in config.xml. +12. Add a $msg.hash.$ filename parameter (md5, sha1, sha256, sha512, optionally truncated with _, e.g. $msg.hash.sha256_16$) that hashes the message payload. Include it in a received-file filename template to dedup by name and content: a re-delivery of the same file overwrites, while changed content gets a new name. See the commented example in partnerships.xml. ## Upgrade Notes See the openAS2HowTo appendix for the general process on upgrading OpenAS2. diff --git a/Server/src/config/partnerships.xml b/Server/src/config/partnerships.xml index 49b43a7d1..6ae49c24e 100644 --- a/Server/src/config/partnerships.xml +++ b/Server/src/config/partnerships.xml @@ -89,6 +89,12 @@ + diff --git a/Server/src/main/java/org/openas2/params/MessageParameters.java b/Server/src/main/java/org/openas2/params/MessageParameters.java index 942ab9d82..cd26c1b1d 100644 --- a/Server/src/main/java/org/openas2/params/MessageParameters.java +++ b/Server/src/main/java/org/openas2/params/MessageParameters.java @@ -6,6 +6,8 @@ import org.openas2.util.Properties; import jakarta.mail.internet.ParseException; +import java.io.InputStream; +import java.security.MessageDigest; import java.util.StringTokenizer; public class MessageParameters extends ParameterParser { @@ -14,6 +16,7 @@ public class MessageParameters extends ParameterParser { public static final String KEY_ATTRIBUTES = "attributes"; public static final String KEY_HEADERS = "headers"; public static final String KEY_CONTENT_FILENAME = "content-disposition"; + public static final String KEY_HASH = "hash"; private Message target; private Logger logger = LoggerFactory.getLogger(MessageParameters.class); @@ -85,11 +88,69 @@ public String getParameter(String key) throws InvalidParameterException { CompositeParameters parser = new CompositeParameters(false).add("date", new DateParameters()).add("msg", new MessageParameters(getTarget())).add("rand", new RandomParameters()); return ParameterParser.parse(filename, parser); } + } else if (area.equals(KEY_HASH)) { + return computeContentHash(areaID); } else { throw new InvalidParameterException("Invalid area in key", this, key, null); } } + /** + * Computes a hex digest of the message payload for use in a filename, e.g. to dedup identical + * content. The spec is the hash algorithm (sha256, sha512, sha1 or md5) optionally followed by + * an underscore and a truncation length, e.g. "sha256_16" for the first 16 hex characters. + */ + private String computeContentHash(String spec) throws InvalidParameterException { + String algorithm = spec; + int length = -1; + int underscore = spec.indexOf('_'); + if (underscore > 0) { + algorithm = spec.substring(0, underscore); + try { + length = Integer.parseInt(spec.substring(underscore + 1)); + } catch (NumberFormatException e) { + throw new InvalidParameterException("Invalid hash truncation length", this, spec, null); + } + } + String jdkAlgorithm; + switch (algorithm.toLowerCase()) { + case "md5": + jdkAlgorithm = "MD5"; + break; + case "sha1": + jdkAlgorithm = "SHA-1"; + break; + case "sha256": + jdkAlgorithm = "SHA-256"; + break; + case "sha512": + jdkAlgorithm = "SHA-512"; + break; + default: + throw new InvalidParameterException("Unsupported hash algorithm (use md5, sha1, sha256 or sha512)", this, spec, null); + } + try { + MessageDigest digest = MessageDigest.getInstance(jdkAlgorithm); + try (InputStream in = getTarget().getData().getInputStream()) { + byte[] buffer = new byte[8192]; + int read; + while ((read = in.read(buffer)) != -1) { + digest.update(buffer, 0, read); + } + } + byte[] hash = digest.digest(); + StringBuilder hex = new StringBuilder(hash.length * 2); + for (byte b : hash) { + hex.append(Character.forDigit((b >> 4) & 0xF, 16)); + hex.append(Character.forDigit(b & 0xF, 16)); + } + String result = hex.toString(); + return (length > 0 && length < result.length()) ? result.substring(0, length) : result; + } catch (Exception e) { + throw new InvalidParameterException("Failed to compute content hash: " + org.openas2.util.Logging.getExceptionMsg(e), this, spec, null); + } + } + public void setTarget(Message message) { target = message; } diff --git a/Server/src/test/java/org/openas2/params/MessageParametersHashTest.java b/Server/src/test/java/org/openas2/params/MessageParametersHashTest.java new file mode 100644 index 000000000..017f90555 --- /dev/null +++ b/Server/src/test/java/org/openas2/params/MessageParametersHashTest.java @@ -0,0 +1,85 @@ +package org.openas2.params; + +import jakarta.mail.internet.InternetHeaders; +import jakarta.mail.internet.MimeBodyPart; +import org.junit.jupiter.api.Test; +import org.openas2.message.AS2Message; +import org.openas2.message.Message; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Verifies the $msg.hash.[_]$ filename parameter used for content-based dedup: + * it hashes the message payload, supports truncation, and gives the same hash for identical content + * and different hashes for different content. + */ +public class MessageParametersHashTest { + + private Message messageWithContent(byte[] content) throws Exception { + InternetHeaders ih = new InternetHeaders(); + ih.setHeader("Content-Type", "application/octet-stream"); + AS2Message msg = new AS2Message(); + msg.setData(new MimeBodyPart(ih, content)); + return msg; + } + + private String expectedHex(byte[] content, String jdkAlgorithm) throws Exception { + byte[] hash = MessageDigest.getInstance(jdkAlgorithm).digest(content); + StringBuilder hex = new StringBuilder(); + for (byte b : hash) { + hex.append(String.format("%02x", b)); + } + return hex.toString(); + } + + @Test + public void fullSha256Hash() throws Exception { + byte[] content = "the quick brown fox".getBytes(StandardCharsets.UTF_8); + MessageParameters params = new MessageParameters(messageWithContent(content)); + + assertEquals(expectedHex(content, "SHA-256"), params.getParameter("hash.sha256")); + } + + @Test + public void truncatedTo16Characters() throws Exception { + byte[] content = "invoice payload 12345".getBytes(StandardCharsets.UTF_8); + MessageParameters params = new MessageParameters(messageWithContent(content)); + + String result = params.getParameter("hash.sha256_16"); + assertEquals(16, result.length(), "hash should be truncated to 16 characters"); + assertEquals(expectedHex(content, "SHA-256").substring(0, 16), result); + } + + @Test + public void identicalContentGivesSameHashAndDifferentContentDiffers() throws Exception { + MessageParameters a = new MessageParameters(messageWithContent("same bytes".getBytes(StandardCharsets.UTF_8))); + MessageParameters b = new MessageParameters(messageWithContent("same bytes".getBytes(StandardCharsets.UTF_8))); + MessageParameters c = new MessageParameters(messageWithContent("other bytes".getBytes(StandardCharsets.UTF_8))); + + assertEquals(a.getParameter("hash.sha256_16"), b.getParameter("hash.sha256_16"), + "identical content must produce the same hash (dedup)"); + assertNotEquals(a.getParameter("hash.sha256_16"), c.getParameter("hash.sha256_16"), + "different content must produce a different hash"); + } + + @Test + public void otherAlgorithmsAreSupported() throws Exception { + byte[] content = "algo test".getBytes(StandardCharsets.UTF_8); + MessageParameters params = new MessageParameters(messageWithContent(content)); + + assertEquals(expectedHex(content, "MD5"), params.getParameter("hash.md5")); + assertEquals(expectedHex(content, "SHA-512"), params.getParameter("hash.sha512")); + } + + @Test + public void unsupportedAlgorithmThrows() throws Exception { + MessageParameters params = new MessageParameters(messageWithContent("x".getBytes(StandardCharsets.UTF_8))); + + assertThrows(InvalidParameterException.class, () -> params.getParameter("hash.crc32")); + } +}