count SNPs in modify

CHANGELOG.md

@@ -1,5 +1,6 @@
 - V 2.3.0.0:
     - Split the functionality of `rectify` into a new subcommand `modify` that exactly mirrors the former `rectify`, and a new `rectify` that just adds and adjusts checksums and increments version numbers to make recently changed package pass validation again.
+    - Added an option `--updateNrSNPs` to `modify` to count the number of non-missing SNPs and add the count to the respective .janno column.
 - V 2.2.2.2:
     - Switched to newer version of sequence-formats, improving genotype data parsing error messages.
     - Changed "PoseidonID" to the correct "Poseidon_ID" in an error message and on the server website.

src-executables/Main-trident.hs

@@ -262,6 +262,7 @@ modifyOptParser = ModifyOptions <$> parseBasePaths
                                 <*> parseChecksumsToRectify
                                 <*> parseMaybeContributors
                                 <*> parseJannoRemoveEmptyCols
+                                <*> parseUpdateNrSNPs
                                 <*> parseOnlyLatest
                                 <*> parseForce
 

src/Poseidon/CLI/Trident/Modify.hs

@@ -5,36 +5,51 @@ module Poseidon.CLI.Trident.Modify (
     updateChecksums, addContributors, completeAndWritePackage
     ) where
 
-import           Poseidon.Core.Contributor     (ContributorSpec (..))
-import           Poseidon.Core.EntityTypes     (HasNameAndVersion (..),
-                                                PacNameAndVersion (..),
-                                                renderNameWithVersion)
-import           Poseidon.Core.GenotypeData    (GenotypeDataSpec (..),
-                                                GenotypeFileSpec (..))
-import           Poseidon.Core.Janno           (makeJannoHeader,
-                                                writeJannoFileWithoutEmptyCols)
-import           Poseidon.Core.Package         (PackageReadOptions (..),
-                                                PoseidonPackage (..),
-                                                defaultPackageReadOptions,
-                                                readPoseidonPackageCollection,
-                                                writePoseidonPackage)
-import           Poseidon.Core.PoseidonVersion (PoseidonVersion (..))
-import           Poseidon.Core.Utils           (PoseidonIO, getChk, logDebug,
-                                                logError, logInfo, logWarning)
-import           Poseidon.Core.Version         (VersionComponent (..),
-                                                updateThreeComponentVersion)
+import           Poseidon.Core.Contributor      (ContributorSpec (..))
+import           Poseidon.Core.EntityTypes      (HasNameAndVersion (..),
+                                                 PacNameAndVersion (..),
+                                                 renderNameWithVersion)
+import           Poseidon.Core.GenotypeData     (GenotypeDataSpec (..),
+                                                 GenotypeFileSpec (..),
+                                                 loadIndividuals,
+                                                 printSNPCopyProgress)
+import           Poseidon.Core.Janno            (JannoRow (..), JannoRows (..),
+                                                 makeJannoHeader,
+                                                 writeJannoFile,
+                                                 writeJannoFileWithoutEmptyCols)
+import           Poseidon.Core.Package          (PackageReadOptions (..),
+                                                 PoseidonException (..),
+                                                 PoseidonPackage (..),
+                                                 defaultPackageReadOptions,
+                                                 getJointGenotypeData,
+                                                 readPoseidonPackageCollection,
+                                                 writePoseidonPackage)
+import           Poseidon.Core.PoseidonVersion  (PoseidonVersion (..))
+import           Poseidon.Core.Utils            (PoseidonIO, envErrorLength,
+                                                 envLogAction, getChk, logDebug,
+                                                 logError, logInfo, logWarning)
+import           Poseidon.Core.Version          (VersionComponent (..),
+                                                 updateThreeComponentVersion)
 
-import           Control.DeepSeq               ((<$!!>))
-import           Control.Monad                 (when)
-import           Control.Monad.IO.Class        (MonadIO, liftIO)
-import           Data.List                     (nub)
-import           Data.Maybe                    (fromJust)
-import           Data.Time                     (UTCTime (..), getCurrentTime)
-import           Data.Version                  (Version (..), makeVersion,
-                                                showVersion)
-import           System.Directory              (doesFileExist, removeFile)
-import           System.Exit                   (exitFailure)
-import           System.FilePath               ((</>))
+import           Control.DeepSeq                ((<$!!>))
+import           Control.Exception              (catch, throwIO)
+import           Control.Monad                  (when)
+import           Control.Monad.IO.Class         (MonadIO, liftIO)
+import           Data.List                      (nub)
+import           Data.Maybe                     (fromJust)
+import           Data.Time                      (UTCTime (..), getCurrentTime)
+import qualified Data.Vector.Unboxed            as VU
+import qualified Data.Vector.Unboxed.Mutable    as VUM
+import           Data.Version                   (Version (..), makeVersion,
+                                                 showVersion)
+import           Pipes                          ((>->))
+import qualified Pipes.Prelude                  as P
+import           Pipes.Safe                     (runSafeT)
+import           Poseidon.CLI.Trident.Forge     (sumNonMissingSNPs)
+import           Poseidon.Core.ColumnTypesJanno (JannoNrSNPs (..))
+import           System.Directory               (doesFileExist, removeFile)
+import           System.Exit                    (exitFailure)
+import           System.FilePath                ((</>))
 
 data ModifyOptions = ModifyOptions
     { _modifyBaseDirs              :: [FilePath]
@@ -44,6 +59,7 @@ data ModifyOptions = ModifyOptions
     , _modifyChecksums             :: ChecksumsToModify
     , _modifyNewContributors       :: Maybe [ContributorSpec]
     , _modifyJannoRemoveEmptyCols  :: Bool
+    , _modifyUpdateNrSNPs          :: Bool
     , _modifyOnlyLatest            :: Bool
     , _modifyForce                 :: Bool
     }
@@ -68,6 +84,7 @@ runModify (ModifyOptions
                 baseDirs
                 ignorePosVer newPosVer pacVerUpdate checksumUpdate newContributors
                 jannoRemoveEmptyCols
+                updateNrSNPs
                 onlyLatest force
            ) = do
     let pacReadOpts = defaultPackageReadOptions {
@@ -100,21 +117,56 @@ runModify (ModifyOptions
         modifyOnePackage :: PoseidonPackage -> PoseidonIO ()
         modifyOnePackage inPac = do
             logInfo $ "Modifying package: " ++ renderNameWithVersion inPac
-            when jannoRemoveEmptyCols $ do
+            -- counting SNPs for .janno column Nr_SNPs
+            let inJannoRows = getJannoRows $ posPacJanno inPac
+            updatedJanno <- if updateNrSNPs
+                            then do
+                                nrSNPs <- countSNPs inPac
+                                nrSNPsFrozen <- liftIO $ VU.freeze nrSNPs
+                                let updatedRows = zipWith
+                                        (\x y -> x {jNrSNPs = Just (JannoNrSNPs y)})
+                                        inJannoRows (VU.toList nrSNPsFrozen)
+                                return $ JannoRows updatedRows
+                            else return $ JannoRows inJannoRows
+            when (updateNrSNPs || jannoRemoveEmptyCols) $
                 case posPacJannoFile inPac of
-                    Nothing   -> do
-                        logWarning "No .janno file to modify with --jannoRemoveEmpty"
-                    Just jannoPath -> do
-                        logInfo "Reordering and removing empty columns from .janno file"
-                        liftIO $ writeJannoFileWithoutEmptyCols
-                                     (posPacBaseDir inPac </> jannoPath)
-                                     (makeJannoHeader (posPacJanno inPac))
-                                     (posPacJanno inPac)
+                        Nothing -> logError "No .janno file to modify"
+                        Just jannoPath -> do
+                            logInfo "Writing .janno file"
+                            if jannoRemoveEmptyCols
+                            then do
+                                logInfo "Reordering and removing empty .janno columns"
+                                liftIO $ writeJannoFileWithoutEmptyCols
+                                             (posPacBaseDir inPac </> jannoPath)
+                                             (makeJannoHeader updatedJanno)
+                                             updatedJanno
+                            else do
+                                liftIO $ writeJannoFile
+                                             (posPacBaseDir inPac </> jannoPath)
+                                             (makeJannoHeader updatedJanno)
+                                             updatedJanno
             updatedPacPosVer <- updatePoseidonVersion newPosVer inPac
             updatedPacContri <- addContributors newContributors updatedPacPosVer
             updatedPacChecksums <- updateChecksums checksumUpdate updatedPacContri
             completeAndWritePackage pacVerUpdate updatedPacChecksums
 
+countSNPs :: PoseidonPackage -> PoseidonIO (VUM.IOVector Int)
+countSNPs pac = do
+    logInfo "Counting SNPs..."
+    inds <- loadIndividuals (posPacBaseDir pac) (posPacGenotypeData pac)
+    logA <- envLogAction
+    currentTime <- liftIO getCurrentTime
+    errLength <- envErrorLength
+    newNrSNPs <- liftIO $ catch (
+        runSafeT $ do
+            eigenstratProd <- getJointGenotypeData logA False False False [pac] Nothing
+            let forgePipe = eigenstratProd >-> printSNPCopyProgress logA currentTime
+            let startAcc = liftIO $ VUM.replicate (length inds) 0
+            P.foldM sumNonMissingSNPs startAcc return forgePipe
+        ) (throwIO . PoseidonGenotypeExceptionForward errLength)
+    logInfo "Done"
+    return newNrSNPs
+
 updatePoseidonVersion :: Maybe Version -> PoseidonPackage -> PoseidonIO PoseidonPackage
 updatePoseidonVersion Nothing    pac = return pac
 updatePoseidonVersion (Just ver) pac = do

src/Poseidon/CLI/Trident/OptparseApplicativeParsers.hs

@@ -251,11 +251,16 @@ readContributorString s = case P.runParser contributorSpecParser () "" s of
     Left p  -> Left (showParsecErr p)
     Right x -> Right x
 
+parseUpdateNrSNPs :: OP.Parser Bool
+parseUpdateNrSNPs = OP.switch (
+    OP.long "updateNrSNPs" <>
+    OP.help "Count the non-missing SNPs per sample and overwrite/fill the Nr_SNPs column in the .janno file."
+    )
+
 parseJannoRemoveEmptyCols :: OP.Parser Bool
 parseJannoRemoveEmptyCols = OP.switch (
     OP.long "jannoRemoveEmpty" <>
-    OP.help "Reorder the .janno file and remove empty colums. \
-            \Remember to pair this option with --checksumJanno to also update the checksum."
+    OP.help "Reorder the .janno file and remove empty colums."
     )
 
 parseMaybeLog :: OP.Parser (Maybe String)

test/PoseidonGoldenTests/GoldenTestCheckSumFile.txt

@@ -70,9 +70,9 @@ e234a2511ce81ff2002b670c5d931b94 rectify rectify/Wang_2020/POSEIDON.yml
 3bb396e099d5b8771a3409f5fe85d70b modify init/Schiffels/CHANGELOG.md
 667074fbd38a002cf44750e92346a118 modify init/Schiffels/POSEIDON.yml
 3bb396e099d5b8771a3409f5fe85d70b modify init/Schiffels/CHANGELOG.md
-ebf0b456d5e09686ee7131db06848104 modify init/Schiffels/POSEIDON.yml
+039b4eccf98334d4295d0eca7de01389 modify init/Schiffels/POSEIDON.yml
 3bb396e099d5b8771a3409f5fe85d70b modify init/Schiffels/CHANGELOG.md
-083fe7ef4206c979356a3a2454d780b1 modify init/Schiffels/Schiffels.janno
+739bfa83c5fc4c6325be34d59078b13a modify init/Schiffels/Schiffels.janno
 5b5c20071f3e53346fdfa04f9dbb0223 forge forge/ForgePac1/POSEIDON.yml
 1286a2580e4bfbed7d804d5f3fe125f7 forge forge/ForgePac1/ForgePac1.geno
 848cf0fab32e4078a88cc17ad8ec3381 forge forge/ForgePac1/ForgePac1.janno
@@ -153,7 +153,7 @@ b4f71aff4fbc11594008c3811781cc43 forge forge/ForgePac22/ForgePac22.bib
 d4a05cfef045648238a94a9d621cf667 chronicle chronicle/chronicle1.yml
 b43da4d5734371c0648553120f812466 timetravel timetravel/Lamnidis_2018-1.0.0/POSEIDON.yml
 8d57ce1a1ab28c0d8a5f391dd790a59c timetravel timetravel/Lamnidis_2018-1.0.1/POSEIDON.yml
-ebf0b456d5e09686ee7131db06848104 timetravel timetravel/Schiffels-1.1.1/POSEIDON.yml
+039b4eccf98334d4295d0eca7de01389 timetravel timetravel/Schiffels-1.1.1/POSEIDON.yml
 1ab24c45ef3a13e0fb34afac7a21dca8 timetravel timetravel/Schmid_2028-1.0.0/POSEIDON.yml
 8d57ce1a1ab28c0d8a5f391dd790a59c fetch fetch/by_package/Lamnidis_2018-1.0.1/POSEIDON.yml
 1ab24c45ef3a13e0fb34afac7a21dca8 fetch fetch/by_package/Schmid_2028-1.0.0/POSEIDON.yml

test/PoseidonGoldenTests/GoldenTestData/chronicle/Schiffels/POSEIDON.yml

@@ -19,7 +19,7 @@ genotypeData:
   indFileChkSum: f77dc756666dbfef3bb35191ae15a167
   snpSet: Other
 jannoFile: Schiffels.janno
-jannoFileChkSum: 083fe7ef4206c979356a3a2454d780b1
+jannoFileChkSum: 739bfa83c5fc4c6325be34d59078b13a
 bibFile: Schiffels.bib
 bibFileChkSum: 9edc4a757f785a8ecb59c54d16c5690a
 changelogFile: CHANGELOG.md

test/PoseidonGoldenTests/GoldenTestData/chronicle/Schiffels/Schiffels.janno

@@ -1,11 +1,11 @@
-Poseidon_ID	Genetic_Sex	Group_Name
-XXX001	M	POP1
-XXX002	F	POP2
-XXX003	M	POP1
-XXX004	F	POP2
-XXX005	M	POP2
-XXX006	F	POP2
-XXX007	M	POP1
-XXX008	F	POP3
-XXX009	F	POP1
-XXX010	M	POP3
+Poseidon_ID	Genetic_Sex	Group_Name	Nr_SNPs
+XXX001	M	POP1	9
+XXX002	F	POP2	9
+XXX003	M	POP1	7
+XXX004	F	POP2	9
+XXX005	M	POP2	7
+XXX006	F	POP2	9
+XXX007	M	POP1	9
+XXX008	F	POP3	9
+XXX009	F	POP1	9
+XXX010	M	POP3	9

test/PoseidonGoldenTests/GoldenTestData/init/Schiffels/POSEIDON.yml

@@ -19,7 +19,7 @@ genotypeData:
   indFileChkSum: f77dc756666dbfef3bb35191ae15a167
   snpSet: Other
 jannoFile: Schiffels.janno
-jannoFileChkSum: 083fe7ef4206c979356a3a2454d780b1
+jannoFileChkSum: 739bfa83c5fc4c6325be34d59078b13a
 bibFile: Schiffels.bib
 bibFileChkSum: 9edc4a757f785a8ecb59c54d16c5690a
 changelogFile: CHANGELOG.md

test/PoseidonGoldenTests/GoldenTestData/init/Schiffels/Schiffels.janno

@@ -1,11 +1,11 @@
-Poseidon_ID	Genetic_Sex	Group_Name
-XXX001	M	POP1
-XXX002	F	POP2
-XXX003	M	POP1
-XXX004	F	POP2
-XXX005	M	POP2
-XXX006	F	POP2
-XXX007	M	POP1
-XXX008	F	POP3
-XXX009	F	POP1
-XXX010	M	POP3
+Poseidon_ID	Genetic_Sex	Group_Name	Nr_SNPs
+XXX001	M	POP1	9
+XXX002	F	POP2	9
+XXX003	M	POP1	7
+XXX004	F	POP2	9
+XXX005	M	POP2	7
+XXX006	F	POP2	9
+XXX007	M	POP1	9
+XXX008	F	POP3	9
+XXX009	F	POP1	9
+XXX010	M	POP3	9

test/PoseidonGoldenTests/GoldenTestData/timetravel/Schiffels-1.1.1/POSEIDON.yml

@@ -19,7 +19,7 @@ genotypeData:
   indFileChkSum: f77dc756666dbfef3bb35191ae15a167
   snpSet: Other
 jannoFile: Schiffels.janno
-jannoFileChkSum: 083fe7ef4206c979356a3a2454d780b1
+jannoFileChkSum: 739bfa83c5fc4c6325be34d59078b13a
 bibFile: Schiffels.bib
 bibFileChkSum: 9edc4a757f785a8ecb59c54d16c5690a
 changelogFile: CHANGELOG.md

test/PoseidonGoldenTests/GoldenTestData/timetravel/Schiffels-1.1.1/Schiffels.janno

@@ -1,11 +1,11 @@
-Poseidon_ID	Genetic_Sex	Group_Name
-XXX001	M	POP1
-XXX002	F	POP2
-XXX003	M	POP1
-XXX004	F	POP2
-XXX005	M	POP2
-XXX006	F	POP2
-XXX007	M	POP1
-XXX008	F	POP3
-XXX009	F	POP1
-XXX010	M	POP3
+Poseidon_ID	Genetic_Sex	Group_Name	Nr_SNPs
+XXX001	M	POP1	9
+XXX002	F	POP2	9
+XXX003	M	POP1	7
+XXX004	F	POP2	9
+XXX005	M	POP2	7
+XXX006	F	POP2	9
+XXX007	M	POP1	9
+XXX008	F	POP3	9
+XXX009	F	POP1	9
+XXX010	M	POP3	9

test/PoseidonGoldenTests/GoldenTestsRunCommands.hs

@@ -709,6 +709,7 @@ testPipelineModify testDir checkFilePath = do
         , _modifyChecksums = ChecksumNone
         , _modifyNewContributors = Nothing
         , _modifyJannoRemoveEmptyCols = False
+        , _modifyUpdateNrSNPs = False
         , _modifyOnlyLatest = False
         , _modifyForce = False
         }
@@ -717,51 +718,33 @@ testPipelineModify testDir checkFilePath = do
           "init" </> "Schiffels" </> "POSEIDON.yml"
         , "init" </> "Schiffels" </> "CHANGELOG.md"
         ]
-    let modifyOpts2 = ModifyOptions {
-          _modifyBaseDirs = [testDir </> "init" </> "Schiffels"]
-        , _modifyPoseidonVersion = Just $ makeVersion [2,7,1]
-        , _modifyIgnorePoseidonVersion = False
+    let modifyOpts2 = modifyOpts1 {
+          _modifyPoseidonVersion = Just $ makeVersion [2,7,1]
         , _modifyPackageVersionUpdate = Just (PackageVersionUpdate Minor (Just "test2"))
         , _modifyChecksums = ChecksumAll
-        , _modifyNewContributors = Nothing
-        , _modifyJannoRemoveEmptyCols = False
-        , _modifyOnlyLatest = False
-        , _modifyForce = False
         }
     let action2 = testLog (runModify modifyOpts2) >> patchLastModified testDir ("init" </> "Schiffels" </> "POSEIDON.yml")
     runAndChecksumFiles checkFilePath testDir action2 "modify" [
           "init" </> "Schiffels" </> "POSEIDON.yml"
         , "init" </> "Schiffels" </> "CHANGELOG.md"
         ]
-    let modifyOpts3 = ModifyOptions {
-          _modifyBaseDirs = [testDir </> "init" </> "Schiffels"]
-        , _modifyPoseidonVersion = Nothing
-        , _modifyIgnorePoseidonVersion = False
-        , _modifyPackageVersionUpdate = Just (PackageVersionUpdate Patch Nothing)
-        , _modifyChecksums = ChecksumNone
+    let modifyOpts3 = modifyOpts1 {
+          _modifyPackageVersionUpdate = Just (PackageVersionUpdate Patch Nothing)
         , _modifyNewContributors = Just [
               ContributorSpec "Josiah Carberry" "carberry@brown.edu" (Just $ ORCID {_orcidNums = "000000021825009", _orcidChecksum = '7'})
             , ContributorSpec "Herbert Testmann" "herbert@testmann.tw" Nothing
             ]
-        , _modifyJannoRemoveEmptyCols = False
-        , _modifyOnlyLatest = False
-        , _modifyForce = False
         }
     let action3 = testLog (runModify modifyOpts3) >> patchLastModified testDir ("init" </> "Schiffels" </> "POSEIDON.yml")
     runAndChecksumFiles checkFilePath testDir action3 "modify" [
           "init" </> "Schiffels" </> "POSEIDON.yml"
         , "init" </> "Schiffels" </> "CHANGELOG.md"
         ]
-    let modifyOpts4 = ModifyOptions {
-          _modifyBaseDirs = [testDir </> "init" </> "Schiffels"]
-        , _modifyPoseidonVersion = Nothing
-        , _modifyIgnorePoseidonVersion = False
-        , _modifyPackageVersionUpdate = Nothing
+    let modifyOpts4 = modifyOpts1 {
+          _modifyPackageVersionUpdate = Nothing
         , _modifyChecksums = ChecksumAll
-        , _modifyNewContributors = Nothing
         , _modifyJannoRemoveEmptyCols = True
-        , _modifyOnlyLatest = False
-        , _modifyForce = False
+        , _modifyUpdateNrSNPs = True
         }
     let action4 = testLog (runModify modifyOpts4) >> patchLastModified testDir ("init" </> "Schiffels" </> "POSEIDON.yml")
     runAndChecksumFiles checkFilePath testDir action4 "modify" [