diff --git a/common-tools/clara-io/src/main/java/org/jlab/io/clara/Clas12Writer.java b/common-tools/clara-io/src/main/java/org/jlab/io/clara/Clas12Writer.java index dabcdb132c..5491d2899b 100644 --- a/common-tools/clara-io/src/main/java/org/jlab/io/clara/Clas12Writer.java +++ b/common-tools/clara-io/src/main/java/org/jlab/io/clara/Clas12Writer.java @@ -8,6 +8,7 @@ import org.jlab.detector.helicity.HelicitySequenceDelayed; import org.jlab.detector.serial.SerialHoncho; import org.jlab.detector.serial.PostProcessor; +import org.jlab.detector.serial.Occupancer; import org.jlab.jnp.hipo4.data.Bank; import org.jlab.jnp.hipo4.data.Event; import org.jlab.jnp.hipo4.data.SchemaFactory; @@ -28,6 +29,7 @@ */ public class Clas12Writer extends HipoToHipoWriter { + Occupancer occupancer; SerialHoncho serial; Bank runConfig; ConstantsManager conman; @@ -35,6 +37,7 @@ public class Clas12Writer extends HipoToHipoWriter { boolean postprocess; private void init(JSONObject opts) { + occupancer = new Occupancer(); fullSchema = new SchemaFactory(); fullSchema.initFromDirectory(FileUtils.getEnvironmentPath("CLAS12DIR","etc/bankdefs/hipo4")); serial = new SerialHoncho(fullSchema); @@ -63,6 +66,7 @@ protected HipoWriterSorted createWriter(Path file, JSONObject opts) throws Event protected void writeEvent(Object event) throws EventWriterException { Event t = serial.read((Event)event); if (!t.isEmpty()) writer.addEvent(t, 1); + occupancer.process(((Event)event)); super.writeEvent(event); } diff --git a/common-tools/clas-detector/src/main/java/org/jlab/detector/serial/Occupancer.java b/common-tools/clas-detector/src/main/java/org/jlab/detector/serial/Occupancer.java new file mode 100644 index 0000000000..92e50b85b1 --- /dev/null +++ b/common-tools/clas-detector/src/main/java/org/jlab/detector/serial/Occupancer.java @@ -0,0 +1,183 @@ +package org.jlab.detector.serial; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Map; +import org.jlab.detector.banks.RawBank; +import org.jlab.utils.groups.IndexedTable; +import org.jlab.utils.groups.IndexedTable.IndexedEntry; +import org.jlab.detector.banks.RawBank.OrderGroups; +import org.jlab.jnp.hipo4.data.Bank; +import org.jlab.jnp.hipo4.data.Event; +import org.jlab.jnp.hipo4.data.Schema; +import org.jlab.jnp.hipo4.data.SchemaFactory; +import org.jlab.utils.system.ClasUtilsFile; + +/** + * Occupancy bookkeeper based on IndexedTable, with I/O helpers for indexed banks. + * + * @author baltzell + */ +public class Occupancer extends ArrayList { + + static final String BANKDIR = ClasUtilsFile.getResourceDir("CLAS12DIR","etc/bankdefs/hipo4/singles/occupancy"); + static final SchemaFactory FULL_SCHEMA = new SchemaFactory(); + static { FULL_SCHEMA.initFromDirectory(ClasUtilsFile.getResourceDir("CLAS12DIR","etc/bankdefs/hipo4")); } + + int prescale = 1000; + int nevents = 0; + + public Occupancer() { + super(); + init(); + } + + public Occupancer(int prescale) { + super(); + this.prescale = prescale; + init(); + } + + private void init() { + try { + // Initialize OccupanceTable list, by reading occupancy bank filenames + // and stripping ".json" suffix and "OCC::" prefix to get hit bank name: + addAll(Files.list(Paths.get(BANKDIR)) + .filter(Files::isRegularFile) + .map(p -> p.getFileName().toString()) + .map(s -> s.substring(0, s.length()-5)) + .map(s -> s.substring(5, s.length())) + .map(OccupanceTable::new).toList()); + } catch (IOException ex) { + System.getLogger(Occupancer.class.getName()).log(System.Logger.Level.ERROR, (String) null, ex); + } + } + + public boolean process(Event event) { + forEach(t -> { t.fill(event, false); }); + if (++nevents % prescale == 0) { + forEach(t -> { + if (t.getTable().getRowCount() > 0) { + event.write(t.create(nevents, event)); + } + t.reset(); + }); + nevents = 0; + } + return true; + } + + public void reset() { + forEach(t -> { t.reset(); nevents = 0; }); + } + + public static class OccupanceTable { + + Schema occSchema; + Schema hitSchema; + IndexedTable table; + + /** + * A 3-index table, e.g., sector/layer/component. + * @param hitBank name of the hit bank + */ + public OccupanceTable(String hitBank) { + hitSchema = FULL_SCHEMA.getSchema(hitBank); + occSchema = FULL_SCHEMA.getSchema("OCC::" + hitBank); + table = new IndexedTable(occSchema.hasEntry("order")?4:3, new String[]{"occ/F"}); + } + + /** + * An N-index table. + * @param hitBank name of the hit bank + * @param indexCount number of inidices in the hit bank + */ + public OccupanceTable(String hitBank, int indexCount) { + hitSchema = FULL_SCHEMA.getSchema(hitBank); + occSchema = FULL_SCHEMA.getSchema("OCC::" + hitBank); + table = new IndexedTable(indexCount, new String[]{"occ/F"}); + } + + public final IndexedTable getTable() { return table; } + + /** + * Zero the occupancy table. + */ + public final void reset() { + table = new IndexedTable(table.getList().getIndexSize(), new String[]{"occ/F"}); + } + + /** + * Get the occupancy table, normalized by number of events. + * @param events + * @return + */ + public final IndexedTable getOccupancy(long events) { + IndexedTable t = new IndexedTable(table.getList().getIndexSize(), new String[]{"occ/F"}); + for (long hash : ((Map)table.getList().getMap()).keySet()) { + t.addEntry(IndexedTable.DEFAULT_GENERATOR.getIndices(hash, table.getList().getIndexSize())); + t.setDoubleValueByHash((table.getDoubleValueByHash(0, hash))/events, 0, hash); + } + return t; + } + + /** + * Fill the occupancy table. + * @param weight + * @param index + */ + public final void fill(float weight, int... index) { + for (int i=0; i m = table.getList().getMap(); + int i = 0; + for (long hash : m.keySet()) { + int[] idx = IndexedTable.DEFAULT_GENERATOR.getIndices(hash, table.getList().getIndexSize()); + for (int j=0; j schemaExempt = Arrays.asList("RUN::config","DC::tdc"); - private CLASDecoder4 decoder = new CLASDecoder4(); + private final CLASDecoder4 decoder = new CLASDecoder4(); + private final Occupancer occupancer = new Occupancer(); public EngineProcessor(){} @@ -134,7 +136,7 @@ public void initDefault(){ String[] names = new String[]{ "MAGFIELDS", "DCCR","DCHB","FTOFHB","EC","HTCC","EBHB", - "DCTB","FTOFTB","EBTB","VTX" + "DCTB","FTOFTB","EBTB","OCC" }; String[] services = new String[]{ @@ -148,7 +150,7 @@ public void initDefault(){ "org.jlab.service.dc.DCTBEngine", "org.jlab.service.ftof.FTOFTBEngine", "org.jlab.service.eb.EBTBEngine", - "org.jlab.rec.service.vtx.VTXEngine" + "org.jlab.calibration.service.OccupanceEngine", }; for(int i = 0; i < names.length; i++){ @@ -164,7 +166,7 @@ public void initAll(){ "CVTFP","CTOF","CND","BAND", "HTCC","LTCC","EBHB", "DCTB","FMT","FTOFTB","CVT","EBTB", - "RICHEB","RTPC","AHDC","ATOF","ALERT", "MC","VTX" + "RICHEB","RTPC","AHDC","ATOF","ALERT", "MC","VTX","OCC" }; String[] services = new String[]{ @@ -197,7 +199,8 @@ public void initAll(){ "org.jlab.service.atof.ATOFEngine", "org.jlab.service.alert.ALERTEngine", "org.jlab.service.mc.TruthMatch", - "org.jlab.rec.service.vtx.VTXEngine" + "org.jlab.rec.service.vtx.VTXEngine", + "org.jlab.calibration.service.OccupanceEngine", }; if(names.length!=services.length) LOGGER.log(Level.SEVERE, "initAll : the list of services does not match the list of service names... "); @@ -316,6 +319,7 @@ public void processFile(String file, String output){ public void processEvent(DataEvent event, HipoDataSync writer) { processEvent(event); removeBanks(event); + occupancer.process(((HipoDataEvent)event).getHipoEvent()); writer.writeEvent(event); } diff --git a/etc/bankdefs/hipo4/occupancy.json b/etc/bankdefs/hipo4/occupancy.json new file mode 100644 index 0000000000..ec02d1efdb --- /dev/null +++ b/etc/bankdefs/hipo4/occupancy.json @@ -0,0 +1,272 @@ +[ + { + "name": "OCC::DC::tdc", + "group": 500, + "item": 10, + "info": "Occupancy Bank", + "entries": [ + { "name":"sector" , "type":"B", "info":"sector"}, + { "name":"layer" , "type":"B", "info":"layer"}, + { "name":"component" , "type":"S", "info":"component"}, + { "name":"hits" , "type":"F", "info":"number of hits"} + ] + }, + { + "name": "OCC::DC::tot", + "group": 500, + "item": 11, + "info": "Occupancy Bank", + "entries": [ + { "name":"sector" , "type":"B", "info":"sector"}, + { "name":"layer" , "type":"B", "info":"layer"}, + { "name":"component" , "type":"S", "info":"component"}, + { "name":"hits" , "type":"F", "info":"number of hits"} + ] + }, + { + "name": "OCC::ECAL::adc", + "group": 500, + "item": 12, + "info": "Occupancy Bank", + "entries": [ + { "name":"sector" , "type":"B", "info":"sector"}, + { "name":"layer" , "type":"B", "info":"layer"}, + { "name":"component" , "type":"S", "info":"component"}, + { "name":"hits" , "type":"F", "info":"number of hits"} + ] + }, + { + "name": "OCC::ECAL::tdc", + "group": 500, + "item": 13, + "info": "Occupancy Bank", + "entries": [ + { "name":"sector" , "type":"B", "info":"sector"}, + { "name":"layer" , "type":"B", "info":"layer"}, + { "name":"component" , "type":"S", "info":"component"}, + { "name":"hits" , "type":"F", "info":"number of hits"} + ] + }, + { + "name": "OCC::FTOF::adc", + "group": 500, + "item": 14, + "info": "Occupancy Bank", + "entries": [ + { "name":"sector" , "type":"B", "info":"sector"}, + { "name":"layer" , "type":"B", "info":"layer"}, + { "name":"component" , "type":"S", "info":"component"}, + { "name":"order" , "type":"B", "info":"order"}, + { "name":"hits" , "type":"F", "info":"number of hits"} + ] + }, + { + "name": "OCC::FTOF::tdc", + "group": 500, + "item": 15, + "info": "Occupancy Bank", + "entries": [ + { "name":"sector" , "type":"B", "info":"sector"}, + { "name":"layer" , "type":"B", "info":"layer"}, + { "name":"component" , "type":"S", "info":"component"}, + { "name":"order" , "type":"B", "info":"order"}, + { "name":"hits" , "type":"F", "info":"number of hits"} + ] + }, + { + "name": "OCC::CTOF::adc", + "group": 500, + "item": 16, + "info": "Occupancy Bank", + "entries": [ + { "name":"sector" , "type":"B", "info":"sector"}, + { "name":"layer" , "type":"B", "info":"layer"}, + { "name":"component" , "type":"S", "info":"component"}, + { "name":"order" , "type":"B", "info":"order"}, + { "name":"hits" , "type":"F", "info":"number of hits"} + ] + }, + { + "name": "OCC::CTOF::tdc", + "group": 500, + "item": 17, + "info": "Occupancy Bank", + "entries": [ + { "name":"sector" , "type":"B", "info":"sector"}, + { "name":"layer" , "type":"B", "info":"layer"}, + { "name":"component" , "type":"S", "info":"component"}, + { "name":"order" , "type":"B", "info":"order"}, + { "name":"hits" , "type":"F", "info":"number of hits"} + ] + }, + { + "name": "OCC::HTCC::adc", + "group": 500, + "item": 18, + "info": "Occupancy Bank", + "entries": [ + { "name":"sector" , "type":"B", "info":"sector"}, + { "name":"layer" , "type":"B", "info":"layer"}, + { "name":"component" , "type":"S", "info":"component"}, + { "name":"hits" , "type":"F", "info":"number of hits"} + ] + }, + { + "name": "OCC::HTCC::tdc", + "group": 500, + "item": 19, + "info": "Occupancy Bank", + "entries": [ + { "name":"sector" , "type":"B", "info":"sector"}, + { "name":"layer" , "type":"B", "info":"layer"}, + { "name":"component" , "type":"S", "info":"component"}, + { "name":"hits" , "type":"F", "info":"number of hits"} + ] + }, + { + "name": "OCC::LTCC::adc", + "group": 500, + "item": 20, + "info": "Occupancy Bank", + "entries": [ + { "name":"sector" , "type":"B", "info":"sector"}, + { "name":"layer" , "type":"B", "info":"layer"}, + { "name":"component" , "type":"S", "info":"component"}, + { "name":"hits" , "type":"F", "info":"number of hits"} + ] + }, + { + "name": "OCC::LTCC::tdc", + "group": 500, + "item": 21, + "info": "Occupancy Bank", + "entries": [ + { "name":"sector" , "type":"B", "info":"sector"}, + { "name":"layer" , "type":"B", "info":"layer"}, + { "name":"component" , "type":"S", "info":"component"}, + { "name":"hits" , "type":"F", "info":"number of hits"} + ] + }, + { + "name": "OCC::BST::adc", + "group": 500, + "item": 22, + "info": "Occupancy Bank", + "entries": [ + { "name":"sector" , "type":"B", "info":"sector"}, + { "name":"layer" , "type":"B", "info":"layer"}, + { "name":"component" , "type":"S", "info":"component"}, + { "name":"hits" , "type":"F", "info":"number of hits"} + ] + }, + { + "name": "OCC::BMT::adc", + "group": 500, + "item": 23, + "info": "Occupancy Bank", + "entries": [ + { "name":"sector" , "type":"B", "info":"sector"}, + { "name":"layer" , "type":"B", "info":"layer"}, + { "name":"component" , "type":"S", "info":"component"}, + { "name":"hits" , "type":"F", "info":"number of hits"} + ] + }, + { + "name": "OCC::FTCAL::adc", + "group": 500, + "item": 24, + "info": "Occupancy Bank", + "entries": [ + { "name":"sector" , "type":"B", "info":"sector"}, + { "name":"layer" , "type":"B", "info":"layer"}, + { "name":"component" , "type":"S", "info":"component"}, + { "name":"hits" , "type":"F", "info":"number of hits"} + ] + }, + { + "name": "OCC::FTHODO::adc", + "group": 500, + "item": 25, + "info": "Occupancy Bank", + "entries": [ + { "name":"sector" , "type":"B", "info":"sector"}, + { "name":"layer" , "type":"B", "info":"layer"}, + { "name":"component" , "type":"S", "info":"component"}, + { "name":"hits" , "type":"F", "info":"number of hits"} + ] + }, + { + "name": "OCC::FTTRK::adc", + "group": 500, + "item": 26, + "info": "Occupancy Bank", + "entries": [ + { "name":"sector" , "type":"B", "info":"sector"}, + { "name":"layer" , "type":"B", "info":"layer"}, + { "name":"component" , "type":"S", "info":"component"}, + { "name":"hits" , "type":"F", "info":"number of hits"} + ] + }, + { + "name": "OCC::RICH::tdc", + "group": 500, + "item": 27, + "info": "Occupancy Bank", + "entries": [ + { "name":"sector" , "type":"B", "info":"sector"}, + { "name":"layer" , "type":"B", "info":"layer"}, + { "name":"component" , "type":"S", "info":"component"}, + { "name":"hits" , "type":"F", "info":"number of hits"} + ] + }, + { + "name": "OCC::BAND::adc", + "group": 500, + "item": 28, + "info": "Occupancy Bank", + "entries": [ + { "name":"sector" , "type":"B", "info":"sector"}, + { "name":"layer" , "type":"B", "info":"layer"}, + { "name":"component" , "type":"S", "info":"component"}, + { "name":"hits" , "type":"F", "info":"number of hits"} + ] + }, + { + "name": "OCC::BAND::tdc", + "group": 500, + "item": 29, + "info": "Occupancy Bank", + "entries": [ + { "name":"sector" , "type":"B", "info":"sector"}, + { "name":"layer" , "type":"B", "info":"layer"}, + { "name":"component" , "type":"S", "info":"component"}, + { "name":"hits" , "type":"F", "info":"number of hits"} + ] + }, + { + "name": "OCC::CND::adc", + "group": 500, + "item": 30, + "info": "Occupancy Bank", + "entries": [ + { "name":"sector" , "type":"B", "info":"sector"}, + { "name":"layer" , "type":"B", "info":"layer"}, + { "name":"component" , "type":"S", "info":"component"}, + { "name":"order" , "type":"B", "info":"order"}, + { "name":"hits" , "type":"F", "info":"number of hits"} + ] + }, + { + "name": "OCC::CND::tdc", + "group": 500, + "item": 31, + "info": "Occupancy Bank", + "entries": [ + { "name":"sector" , "type":"B", "info":"sector"}, + { "name":"layer" , "type":"B", "info":"layer"}, + { "name":"component" , "type":"S", "info":"component"}, + { "name":"order" , "type":"B", "info":"order"}, + { "name":"hits" , "type":"F", "info":"number of hits"} + ] + } +] diff --git a/etc/bankdefs/util/bankSplit.py b/etc/bankdefs/util/bankSplit.py index 52b134174c..591ebad0fd 100755 --- a/etc/bankdefs/util/bankSplit.py +++ b/etc/bankdefs/util/bankSplit.py @@ -1,8 +1,9 @@ #!/usr/bin/env python -import json -import sys import os +import sys +import json +import glob # print usage if len(sys.argv)<2: @@ -54,6 +55,9 @@ def create(dirname, banklist): print("Single json files saved in " + workdirectory + singledirectory) +# occupancy banks: +occ = [os.path.basename(x)[:-5] for x in glob.glob('./singles/occupancy/*.json')] + # these should *always* be kept: mc = ["MC::Event", "MC::GenMatch", "MC::Header", "MC::Lund", "MC::Particle", "MC::RecMatch", "MC::True"] tag1 = ["RUN::config", "RAW::epics", "RAW::scaler", "RUN::scaler", "HEL::scaler", "COAT::config", "HEL::flip", "HEL::online", "HEL::decoder"] @@ -83,7 +87,7 @@ def create(dirname, banklist): trig = ["RAW::vtp","HTCC::rec","ECAL::adc","ECAL::calib","ECAL::clusters","ECAL::hits","ECAL::moments","ECAL::peaks","ECAL::tdc","ECAL::trigger"] # accumulate all the DST banks: -dst = rectbai + rectb + mc + tag1 + dets +dst = rectbai + rectb + mc + tag1 + occ + dets dsthb = dst + rechbai + rechb # generate the calib and mon schema: @@ -112,8 +116,6 @@ def create(dirname, banklist): level3 = list(dst) level3.extend(["DC::tdc", "DC::tot", "ECAL::adc", "ECAL::clusters", "FTOF::tdc", "FTOF::adc", "HitBasedTrkg::HBClusters", "HitBasedTrkg::HBTracks", "HTCC::adc", "RF::adc", "RF::tdc", "RUN::rf", "TimeBasedTrkg::TBClusters", "TimeBasedTrkg::TBTracks"]) -import glob - rgl = glob.glob('./singles/full/ALERT*.json') rgl.extend(glob.glob('./singles/full/AHDC*.json')) rgl.extend(glob.glob('./singles/full/ATOF*.json'))