MadtRex: The really big PR

.github/gpu_ci/madmatrix_checks.py

@@ -26,6 +26,13 @@
    synchronise with the default stream) gives the same |M|^2 and helicities as on the
    default stream: every memset, kernel and copy of the matrix element must go to the
    stream umami is given, the helicity filtering of the first call included.
+4. helicity_index: ``g g > t t~ z`` (massless and massive vectors, a longitudinal Z,
+   massive fermions) evaluated for one given helicity per event (UMAMI_IN_HELICITY_INDEX,
+   --hel-index), on both backends: the |M|^2 of the ncomb helicities, each given to every
+   event, add up to the helicity-summed |M|^2, the GPU ones equal the CPU ones helicity by
+   helicity, and with a random helicity per event (the threads of a warp, or the lanes of
+   a SIMD vector, on different helicities in lock-step) every event gets the |M|^2 of its
+   helicity and returns it as the chosen helicity.
 
 Any CPU backend but scalar (simd_128, ...) works too, to try the script without a GPU;
 check 3 is then skipped. Writes summary.txt (key=value lines) in the current directory;
@@ -46,7 +53,7 @@
 
 GPU_BACKENDS = ('cuda', 'hip')
 CPU_BACKEND = 'scalar'
-CHECKS = ('mssm_gpu_vs_cpu', 'zero_me', 'nonblocking_stream')
+CHECKS = ('mssm_gpu_vs_cpu', 'zero_me', 'nonblocking_stream', 'helicity_index')
 
 results = {}
 
@@ -66,7 +73,7 @@ def run(cmd, cwd=None, log=None):
 
 
 def write(path, typecode, values):
-    """A raw binary file of check_sa.exe: 'd' double, 'I' 32-bit unsigned int."""
+    """A raw binary file of check_sa.exe: 'd' double, 'I'/'i' 32-bit unsigned/signed int."""
     with open(path, 'wb') as f:
         array.array(typecode, values).tofile(f)
     return os.path.abspath(path)
@@ -267,6 +274,70 @@ def check_nonblocking_stream(backend, repo, ncalls=4):
           '%d calls of %d events identical to the default stream' % (ncalls, nevt // ncalls))
 
 
+# ---------------------------------------------------------------------------
+# 4. one given helicity per event
+# ---------------------------------------------------------------------------
+
+def check_helicity_index(backend, repo, nevt=512):
+    section('g g > t t~ z, one given helicity per event (%s, %s)' % (backend, CPU_BACKEND))
+    name = 'ggttxz_hel'
+    (proc_dir,) = output(repo, name, 'sm', 'g g > t t~ z', (CPU_BACKEND, backend))
+    data = open(os.path.join(proc_dir, 'ProcessData.h')).read()
+    ncomb = int(re.search(r'constexpr int ncomb = (\d+);', data).group(1))
+    batch = [1, nevt, 1]
+    momenta = dump(name, 'momenta')
+    check_sa(proc_dir, CPU_BACKEND, ['--energy', 2000, '--dump-momenta', momenta] + batch,
+             name + '_rambo.log')
+    rng = random.Random(17)
+    mixed = [rng.randrange(ncomb) for _ in range(nevt)]
+    problems, details = [], []
+    perhel = {}
+    for run_backend in (CPU_BACKEND, backend):
+        prefix = '%s_%s' % (name, run_backend)
+        check_sa(proc_dir, run_backend, ['--momenta', momenta, '--dump-me', dump(prefix, 'me')] + batch,
+                 prefix + '.log')
+        summed = read(dump(prefix, 'me'), 'd')
+        perhel[run_backend] = []
+        for ihel in range(ncomb):
+            hprefix = '%s_h%d' % (prefix, ihel)
+            hels = write(hprefix + '_in.bin', 'i', [ihel] * nevt)
+            check_sa(proc_dir, run_backend, ['--momenta', momenta, '--hel-index', hels,
+                                             '--dump-me', dump(hprefix, 'me'),
+                                             '--dump-hel', dump(hprefix, 'hel')] + batch,
+                     hprefix + '.log')
+            perhel[run_backend].append(read(dump(hprefix, 'me'), 'd'))
+            if set(read(dump(hprefix, 'hel'), 'i')) != {ihel}:
+                problems.append('%s: helicity %d not returned as the chosen helicity' % (run_backend, ihel))
+        total = [sum(perhel[run_backend][ihel][ievt] for ihel in range(ncomb)) for ievt in range(nevt)]
+        rels = [abs(a - b) / abs(b) if b != 0 else float('inf') for a, b in zip(total, summed)]
+        bad = [ievt for ievt, rel in enumerate(rels) if not rel < 1e-10]
+        if bad or len(summed) != nevt:
+            problems.append('%s: the sum over the helicities differs from the helicity sum in %d of %d events'
+                            % (run_backend, len(bad), nevt))
+        # a random helicity per event, the same events
+        mprefix = '%s_mixed' % prefix
+        hels = write(mprefix + '_in.bin', 'i', mixed)
+        check_sa(proc_dir, run_backend, ['--momenta', momenta, '--hel-index', hels,
+                                         '--dump-me', dump(mprefix, 'me'),
+                                         '--dump-hel', dump(mprefix, 'hel')] + batch,
+                 mprefix + '.log')
+        me, hel = read(dump(mprefix, 'me'), 'd'), read(dump(mprefix, 'hel'), 'i')
+        expected = [perhel[run_backend][ihel][ievt] for ievt, ihel in enumerate(mixed)]
+        bad = [ievt for ievt, (a, b) in enumerate(zip(me, expected)) if not abs(a - b) <= 1e-12 * abs(b)]
+        if bad or list(hel) != mixed:
+            problems.append('%s: a random helicity per event: %d |M|^2 differ from that helicity\'s, '
+                            '%d helicities not returned' % (run_backend, len(bad),
+                                                            sum(1 for a, b in zip(hel, mixed) if a != b)))
+        details.append('%s: max rel diff of the helicity sum %.1e' % (run_backend, max(rels, default=0.)))
+        print('  ' + details[-1])
+    bad = sum(1 for ihel in range(ncomb) for a, b in zip(perhel[backend][ihel], perhel[CPU_BACKEND][ihel])
+              if not abs(a - b) <= 1e-9 * max(abs(b), 1e-300))
+    if bad:
+        problems.append('%d (helicity, event) |M|^2 differ between %s and %s' % (bad, backend, CPU_BACKEND))
+    check('helicity_index', not problems, '; '.join(problems[:3]) or
+          '%d helicities x %d events; %s' % (ncomb, nevt, '; '.join(details)))
+
+
 def main():
     parser = argparse.ArgumentParser(description=__doc__.split('\n')[0])
     parser.add_argument('--repo', required=True)

.github/workflows/acceptancetest_mg7.yml

@@ -564,6 +564,39 @@ jobs:
             ./tests/test_manager.py test_systematics_mg7 -pA -t0 -l INFO
 
 
+  acceptancetest_mg7_madtrex:
+    needs: build_madspace
+    # MadtRex (madtrex/, 'install madtrex'): built on top of the cached madspace
+    # install, then its tests, which all skip without an installed madtrex (and
+    # a skipped test fails the step, so a broken install cannot pass quietly):
+    # test_madtrex_standalone (the standalone driver/bin/reweight, the card
+    # readers, in-place weights) and test_reweight_backends (classic module and
+    # MadtRex reweight the same events and agree event by event; needs f2py).
+    runs-on: ubuntu-24.04
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.fork == true
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/checkout_mg5
+      - uses: ./.github/actions/install_madspace
+      - uses: ./.github/actions/restore_heptools
+      - uses: ./.github/actions/restore-pip-cache
+      - name: install madtrex
+        run: |
+            cd $GITHUB_WORKSPACE
+            python3 madtrex/install.py --yes -j 4
+      - name: test_madtrex_standalone
+        if: ${{ !cancelled() }}
+        run: |
+            cd $GITHUB_WORKSPACE
+            ./tests/test_manager.py test_madtrex_standalone -t0
+      - name: test_reweight_backends
+        if: ${{ !cancelled() }}
+        run: |
+            cd $GITHUB_WORKSPACE
+            export PATH="$HOME/.cache/HEPtools/bin:$PATH"
+            env -u LHAPDF_DATA_PATH ./tests/test_manager.py test_reweight_backends -pA -t0 -l INFO
+
+
   acceptancetest_mg7_uncovered:
     # mg7 acceptance tests that no workflow selected (audit of 2026-10-24).
     # test_mg7_custom_functions and test_mg7_interference compare to runs with

.github/workflows/unittest.yml

@@ -324,6 +324,17 @@ jobs:
             cd $GITHUB_WORKSPACE
             ./tests/test_manager.py test_lhe_ordering test_reweight_ordering test_boost_by_a_momentum_at_rest test_nested_identical_children -t0
 
+      # ---- unittest_reweight_backend ----
+      # the reweighting backend choice (classic module or MadtRex) and the
+      # MadtRex reweight-card readers: pure Python, no madspace/madtrex. The
+      # tests needing an installed madtrex run in acceptancetest_mg7.yml
+      # (acceptancetest_mg7_madtrex).
+      - name: "unittest_reweight_backend: reweighting backend choice and MadtRex cards"
+        if: ${{ !cancelled() }}
+        run: |
+            cd $GITHUB_WORKSPACE
+            ./tests/test_manager.py test_reweight_backend NpyToLheNamedWeightsTest HelicityReweightingCardTest ClassicVocabularyTest GridpackSourceBuildTest -t0
+
       # Leave /tmp empty for the next job on this self-hosted machine.
       - if: always()
         uses: ./.github/actions/clean_tmp

MadSpin/interface_madspin.py

@@ -12064,7 +12064,7 @@ def _density_libtag(self, tree, prod_or_decay):
         that build's tag."""
         tag = madspin.density_libtag(self._ms_run_id,
                                      'p' if prod_or_decay == 'prod' else 'd')
-        if os.path.exists(pjoin(tree, 'lib', 'libmadmatrix_common%s_scalar.so' % tag)):
+        if os.path.exists(madmatrix_density.common_library_path(tree, tag)):
             return tag
         try:
             return madmatrix_density.find_libtag(tree)

UpdateNotes.txt

@@ -36,6 +36,42 @@ ANNOUNCEMENT:
        NNPDF40_nlo_as_01180_qed (lhaid 335900), at NLO also for a model with a
        massive b (NNPDF4.0 has no 4-flavour QED set). The NNPDF4.0 defaults
        have no photon. A PDF set by the user is kept.
+   ZW: MG7 outputs have two reweighting backends: MadtRex ('install madtrex';
+       matrix-element and PDF/scale reweighting on lhe or lhe_npy events) and the
+       classic reweighting module (f2py). Both read Cards/reweight_card.dat with the
+       classic vocabulary ('change' for options, 'set' for parameters). The backend
+       comes from 'generate_events --reweight-backend=classic|madtrex|auto', the
+       reweight switch of the launch question (now OFF/ON/classic/madtrex/density,
+       ON being auto), or the card's new 'change backend' line; auto prefers
+       MadtRex when it can run the card. The choice is logged and recorded in
+       Events/<run>/reweighting.json, and a choice that cannot run stops the launch
+       before the generation. Reweighting runs once, after the systematics;
+       '--reweight=CARD' now just switches it on with CARD as the reweight card.
+       MadtRex cards: 'change helicity', 'change fptype', 'change mode LO|pdf',
+       'change verbose/max_batch/...' replace the former 'set' spellings (still
+       accepted, with a warning), 'set MT 175' (a parameter by its param_card
+       name) works, and directives only the classic module knows are an error
+       instead of being ignored. MG7 outputs get their own default reweight card
+       documenting both backends. madevent and aMC@NLO outputs are unchanged.
+       Both backends add their weights to the run's own event file
+       (Events/<run>/events.lhe.gz, or events.npy for an lhe_npy run reweighted by
+       MadtRex, as rwgt_<name> columns that npy_to_lhe carries over), after the
+       systematics weights; MadtRex no longer writes a separate *_reweighted file
+       (bin/reweight --output still can). MadtRex reads lhe_npy events with
+       systematics weights (they were refused as "Invalid header for event file").
+   ZW: Without f2py_compiler set, the f2py installed with the running Python (e.g.
+       in a virtual environment) is used before f2py3/f2py on the PATH, which may
+       build modules for another Python version (reweighting then failed with "file
+       too short" or a missing all_matrix2py module).
+   OM: MadtRex follow-ups: 'install madtrex' builds with clang/macOS; madtrex refuses a
+       madspace other than the one it was built against (with the command to rebuild it;
+       MadGraph then reweights with the classic module) and its installer a too old
+       madspace; a failing reweighting no longer stops the run, and 'reweight RUN' (after
+       'launch PROC -i') reweights the events of an earlier run; MadtRex reweight cards
+       accept '!' comments and multi-index parameters ('set NMIX 1 2 0.5'); the baseline
+       PDF set and precision of a MadtRex reweighting are those of the run that made the
+       events (its LHE header), not of the current run_card; a gridpack made with
+       include_madtrex_source builds madtrex from its sources in bin/reweight.
    OM: new command 'install cuttools_iregi' recompiling CutTools and IREGI (vendor/)
        with the current fortran compiler. The error raised when CutTools/IREGI were
        compiled with another compiler (e.g. after a gfortran update) now points to it.