Add $msg.hash filename parameter for content-based dedup of received files

RELEASE-NOTES.md

@@ -25,7 +25,8 @@ This is a minor enhancement and bugfix release.
 9. Add DbPartnershipFactory: partnerships can optionally be stored in a database (Azure SQL, PostgreSQL, MySQL, Oracle or the embedded H2) instead of the partnerships XML file. See the commented example in config.xml. The required tables are included in db_ddl.sql and openas2-schema.xml and are only needed when using the database partnership store.
 10. Add mutual TLS (client certificate) authentication for outbound HTTPS connections using the https_client_keystore, https_client_keystore_password and https_client_cert_alias partnership attributes (or properties for a global client identity). See the commented example in partnerships.xml.
 11. Add JmsPollingModule: an alternative outbound intake that consumes work from an AMQP 1.0 message queue (Azure Service Bus or any AMQP broker via Apache Qpid JMS) instead of polling a directory. An external producer publishes a queue message identifying the sender/receiver AS2 IDs and the file path; the file is sent through the existing pipeline and the broker owns retry/dead-lettering. See the commented example in config.xml.
-12. Record the stored MDN file path in the message tracking database (new mdn_file_path column) and add a messages/mdnpath API command that returns the MDN file path for a message given its payload filename (GET /api/messages/mdnpath/<filename>). If you use the DB tracking module with an existing external database, add the new column: ALTER TABLE msg_metadata ADD COLUMN mdn_file_path LONGVARCHAR (or the equivalent for your database).
+12. Add a $msg.hash.<algorithm>$ filename parameter (md5, sha1, sha256, sha512, optionally truncated with _<length>, e.g. $msg.hash.sha256_16$) that hashes the message payload. Include it in a received-file filename template to dedup by name and content: a re-delivery of the same file overwrites, while changed content gets a new name. See the commented example in partnerships.xml.
+13. Record the stored MDN file path in the message tracking database (new mdn_file_path column) and add a messages/mdnpath API command that returns the MDN file path for a message given its payload filename (GET /api/messages/mdnpath/<filename>). If you use the DB tracking module with an existing external database, add the new column: ALTER TABLE msg_metadata ADD COLUMN mdn_file_path LONGVARCHAR (or the equivalent for your database).
 
 ## Upgrade Notes
  See the openAS2HowTo appendix for the general process on upgrading OpenAS2.

Server/src/config/partnerships.xml

@@ -89,6 +89,12 @@
         <sender name="PartnerA"/>
         <receiver name="MyCompany"/>
         <attribute name="store_received_file_to" value="$properties.storageBaseDir$/inbox/$msg.receiver.as2_id$/inbox/$msg.sender.as2_id$-$rand.12345$-$msg.content-disposition.filename$"/>
+        <!-- To dedup received files by name AND content, include a hash of the payload in the file
+             name instead of $rand$. A re-delivery of the same file (same name and bytes) then resolves
+             to the same path and overwrites, while a changed file gets a new name. $msg.hash.<algo>$
+             supports md5, sha1, sha256 and sha512, optionally truncated with _<length>, for example:
+        <attribute name="store_received_file_to" value="$properties.storageBaseDir$/inbox/$msg.receiver.as2_id$/inbox/$msg.content-disposition.filename$-$msg.hash.sha256_16$"/>
+        -->
         <attribute name="reject_unsigned_messages" value="true"/>
     </partnership>
 

Server/src/main/java/org/openas2/params/MessageParameters.java

@@ -6,6 +6,8 @@
 import org.openas2.util.Properties;
 
 import jakarta.mail.internet.ParseException;
+import java.io.InputStream;
+import java.security.MessageDigest;
 import java.util.StringTokenizer;
 
 public class MessageParameters extends ParameterParser {
@@ -14,6 +16,7 @@ public class MessageParameters extends ParameterParser {
     public static final String KEY_ATTRIBUTES = "attributes";
     public static final String KEY_HEADERS = "headers";
     public static final String KEY_CONTENT_FILENAME = "content-disposition";
+    public static final String KEY_HASH = "hash";
     private Message target;
 
     private Logger logger = LoggerFactory.getLogger(MessageParameters.class);
@@ -85,11 +88,69 @@ public String getParameter(String key) throws InvalidParameterException {
                 CompositeParameters parser = new CompositeParameters(false).add("date", new DateParameters()).add("msg", new MessageParameters(getTarget())).add("rand", new RandomParameters());
                 return ParameterParser.parse(filename, parser);
             }
+        } else if (area.equals(KEY_HASH)) {
+            return computeContentHash(areaID);
         } else {
             throw new InvalidParameterException("Invalid area in key", this, key, null);
         }
     }
 
+    /**
+     * Computes a hex digest of the message payload for use in a filename, e.g. to dedup identical
+     * content. The spec is the hash algorithm (sha256, sha512, sha1 or md5) optionally followed by
+     * an underscore and a truncation length, e.g. "sha256_16" for the first 16 hex characters.
+     */
+    private String computeContentHash(String spec) throws InvalidParameterException {
+        String algorithm = spec;
+        int length = -1;
+        int underscore = spec.indexOf('_');
+        if (underscore > 0) {
+            algorithm = spec.substring(0, underscore);
+            try {
+                length = Integer.parseInt(spec.substring(underscore + 1));
+            } catch (NumberFormatException e) {
+                throw new InvalidParameterException("Invalid hash truncation length", this, spec, null);
+            }
+        }
+        String jdkAlgorithm;
+        switch (algorithm.toLowerCase()) {
+        case "md5":
+            jdkAlgorithm = "MD5";
+            break;
+        case "sha1":
+            jdkAlgorithm = "SHA-1";
+            break;
+        case "sha256":
+            jdkAlgorithm = "SHA-256";
+            break;
+        case "sha512":
+            jdkAlgorithm = "SHA-512";
+            break;
+        default:
+            throw new InvalidParameterException("Unsupported hash algorithm (use md5, sha1, sha256 or sha512)", this, spec, null);
+        }
+        try {
+            MessageDigest digest = MessageDigest.getInstance(jdkAlgorithm);
+            try (InputStream in = getTarget().getData().getInputStream()) {
+                byte[] buffer = new byte[8192];
+                int read;
+                while ((read = in.read(buffer)) != -1) {
+                    digest.update(buffer, 0, read);
+                }
+            }
+            byte[] hash = digest.digest();
+            StringBuilder hex = new StringBuilder(hash.length * 2);
+            for (byte b : hash) {
+                hex.append(Character.forDigit((b >> 4) & 0xF, 16));
+                hex.append(Character.forDigit(b & 0xF, 16));
+            }
+            String result = hex.toString();
+            return (length > 0 && length < result.length()) ? result.substring(0, length) : result;
+        } catch (Exception e) {
+            throw new InvalidParameterException("Failed to compute content hash: " + org.openas2.util.Logging.getExceptionMsg(e), this, spec, null);
+        }
+    }
+
     public void setTarget(Message message) {
         target = message;
     }

Server/src/test/java/org/openas2/params/MessageParametersHashTest.java

@@ -0,0 +1,85 @@
+package org.openas2.params;
+
+import jakarta.mail.internet.InternetHeaders;
+import jakarta.mail.internet.MimeBodyPart;
+import org.junit.jupiter.api.Test;
+import org.openas2.message.AS2Message;
+import org.openas2.message.Message;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * Verifies the $msg.hash.<algorithm>[_<length>]$ filename parameter used for content-based dedup:
+ * it hashes the message payload, supports truncation, and gives the same hash for identical content
+ * and different hashes for different content.
+ */
+public class MessageParametersHashTest {
+
+    private Message messageWithContent(byte[] content) throws Exception {
+        InternetHeaders ih = new InternetHeaders();
+        ih.setHeader("Content-Type", "application/octet-stream");
+        AS2Message msg = new AS2Message();
+        msg.setData(new MimeBodyPart(ih, content));
+        return msg;
+    }
+
+    private String expectedHex(byte[] content, String jdkAlgorithm) throws Exception {
+        byte[] hash = MessageDigest.getInstance(jdkAlgorithm).digest(content);
+        StringBuilder hex = new StringBuilder();
+        for (byte b : hash) {
+            hex.append(String.format("%02x", b));
+        }
+        return hex.toString();
+    }
+
+    @Test
+    public void fullSha256Hash() throws Exception {
+        byte[] content = "the quick brown fox".getBytes(StandardCharsets.UTF_8);
+        MessageParameters params = new MessageParameters(messageWithContent(content));
+
+        assertEquals(expectedHex(content, "SHA-256"), params.getParameter("hash.sha256"));
+    }
+
+    @Test
+    public void truncatedTo16Characters() throws Exception {
+        byte[] content = "invoice payload 12345".getBytes(StandardCharsets.UTF_8);
+        MessageParameters params = new MessageParameters(messageWithContent(content));
+
+        String result = params.getParameter("hash.sha256_16");
+        assertEquals(16, result.length(), "hash should be truncated to 16 characters");
+        assertEquals(expectedHex(content, "SHA-256").substring(0, 16), result);
+    }
+
+    @Test
+    public void identicalContentGivesSameHashAndDifferentContentDiffers() throws Exception {
+        MessageParameters a = new MessageParameters(messageWithContent("same bytes".getBytes(StandardCharsets.UTF_8)));
+        MessageParameters b = new MessageParameters(messageWithContent("same bytes".getBytes(StandardCharsets.UTF_8)));
+        MessageParameters c = new MessageParameters(messageWithContent("other bytes".getBytes(StandardCharsets.UTF_8)));
+
+        assertEquals(a.getParameter("hash.sha256_16"), b.getParameter("hash.sha256_16"),
+                "identical content must produce the same hash (dedup)");
+        assertNotEquals(a.getParameter("hash.sha256_16"), c.getParameter("hash.sha256_16"),
+                "different content must produce a different hash");
+    }
+
+    @Test
+    public void otherAlgorithmsAreSupported() throws Exception {
+        byte[] content = "algo test".getBytes(StandardCharsets.UTF_8);
+        MessageParameters params = new MessageParameters(messageWithContent(content));
+
+        assertEquals(expectedHex(content, "MD5"), params.getParameter("hash.md5"));
+        assertEquals(expectedHex(content, "SHA-512"), params.getParameter("hash.sha512"));
+    }
+
+    @Test
+    public void unsupportedAlgorithmThrows() throws Exception {
+        MessageParameters params = new MessageParameters(messageWithContent("x".getBytes(StandardCharsets.UTF_8)));
+
+        assertThrows(InvalidParameterException.class, () -> params.getParameter("hash.crc32"));
+    }
+}