diff --git a/CHANGELOG.md b/CHANGELOG.md index 6549e1ed9..cf4b13d54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ +- V 2.3.0.0: + - Split the functionality of `rectify` into a new subcommand `modify` that exactly mirrors the former `rectify`, and a new `rectify` that just adds and adjusts checksums and increments version numbers to make recently changed package pass validation again. - V 2.2.2.2: - - uses newer version of sequence-formats, improving + - Switched to newer version of sequence-formats, improving genotype data parsing error messages. - Changed "PoseidonID" to the correct "Poseidon_ID" in an error message and on the server website. - V 2.2.2.1: - Update of the sequence-formats dependency because of a purely technical build issue on windows, caused by a non-standard directory name. diff --git a/poseidon-hs.cabal b/poseidon-hs.cabal index 6ad6f8d4c..6780402a6 100644 --- a/poseidon-hs.cabal +++ b/poseidon-hs.cabal @@ -1,5 +1,5 @@ name: poseidon-hs -version: 2.2.2.2 +version: 2.3.0.0 synopsis: A package with tools for working with Poseidon genotype data description: The tools in this package read and analyse Poseidon-formatted genotype databases, a modular system for storing genotype data from thousands of individuals. license: MIT @@ -28,7 +28,7 @@ library Poseidon.CLI.Trident.Survey, Poseidon.CLI.Trident.Forge, Poseidon.CLI.Trident.Init, Poseidon.CLI.Trident.Rectify, Poseidon.CLI.Trident.Fetch, Poseidon.CLI.Trident.Genoconvert, Poseidon.CLI.Trident.OptparseApplicativeParsers, Poseidon.CLI.Trident.Timetravel, - Poseidon.CLI.Trident.Jannocoalesce, + Poseidon.CLI.Trident.Jannocoalesce, Poseidon.CLI.Trident.Modify -- analysis Poseidon.Analysis.FStatsConfig, Poseidon.Analysis.RASconfig, Poseidon.Analysis.Utils, Poseidon.Analysis.Generator.Parsers, Poseidon.Analysis.Generator.Types, @@ -40,7 +40,7 @@ library build-depends: base >= 4.7 && < 5, sequence-formats>=1.6.1, text, time, pipes-safe, foldl, exceptions, pipes, bytestring, filepath, yaml, aeson, directory, parsec, vector, pipes-ordered-zip, table-layout<1.0.0.0, mtl, split, warp, warp-tls, wai-cors, - scotty, cassava, pureMD5, wai, githash, attoparsec, pipes-group, + scotty, cassava, cryptohash-md5, base16-bytestring, wai, githash, attoparsec, pipes-group, http-conduit, conduit, http-types, zip-stream, resourcet, MonadRandom, lens-family, unordered-containers, network-uri, optparse-applicative, co-log, regex-tdfa, scientific, country, generics-sop, containers, process, deepseq, template-haskell, diff --git a/src-executables/Main-trident.hs b/src-executables/Main-trident.hs index 94a272cbe..27fc70d65 100644 --- a/src-executables/Main-trident.hs +++ b/src-executables/Main-trident.hs @@ -15,6 +15,8 @@ import Poseidon.CLI.Trident.Jannocoalesce (JannoCoalesceO runJannocoalesce) import Poseidon.CLI.Trident.List (ListOptions (..), runList) +import Poseidon.CLI.Trident.Modify (ModifyOptions (..), + runModify) import Poseidon.CLI.Trident.OptparseApplicativeParsers import Poseidon.CLI.Trident.Rectify (RectifyOptions (..), runRectify) @@ -69,6 +71,7 @@ data Subcommand = | CmdSummarise SummariseOptions | CmdSurvey SurveyOptions | CmdRectify RectifyOptions + | CmdModify ModifyOptions | CmdValidate ValidateOptions | CmdChronicle ChronicleOptions | CmdTimetravel TimetravelOptions @@ -98,6 +101,7 @@ runCmd o = case o of CmdInit opts -> runInit opts CmdList opts -> runList opts CmdRectify opts -> runRectify opts + CmdModify opts -> runModify opts CmdServe opts -> runServerMainThread opts CmdSummarise opts -> runSummarise opts CmdSurvey opts -> runSurvey opts @@ -137,6 +141,7 @@ subcommandParser = OP.subparser ( OP.command "genoconvert" genoconvertOptInfo <> OP.command "jannocoalesce" jannocoalesceOptInfo <> OP.command "rectify" rectifyOptInfo <> + OP.command "modify" modifyOptInfo <> OP.commandGroup "Package creation and manipulation commands:" ) <|> OP.subparser ( @@ -179,6 +184,8 @@ subcommandParser = OP.subparser ( )) rectifyOptInfo = OP.info (OP.helper <*> (CmdRectify <$> rectifyOptParser)) (OP.progDesc "Adjust POSEIDON.yml files automatically to package changes") + modifyOptInfo = OP.info (OP.helper <*> (CmdModify <$> modifyOptParser)) + (OP.progDesc "Modify certain elements of Poseidon packages") validateOptInfo = OP.info (OP.helper <*> (CmdValidate <$> validateOptParser)) (OP.progDesc "Check Poseidon packages or package components for structural correctness") chronicleOptInfo = OP.info (OP.helper <*> (CmdChronicle <$> chronicleOptParser)) @@ -244,13 +251,19 @@ surveyOptParser = SurveyOptions <$> parseBasePaths rectifyOptParser :: OP.Parser RectifyOptions rectifyOptParser = RectifyOptions <$> parseBasePaths - <*> parseIgnorePoseidonVersion - <*> parseMaybePoseidonVersion <*> parseMaybePackageVersionUpdate - <*> parseChecksumsToRectify <*> parseMaybeContributors - <*> parseJannoRemoveEmptyCols - <*> parseOnlyLatest + +modifyOptParser :: OP.Parser ModifyOptions +modifyOptParser = ModifyOptions <$> parseBasePaths + <*> parseIgnorePoseidonVersion + <*> parseMaybePoseidonVersion + <*> parseMaybePackageVersionUpdate + <*> parseChecksumsToRectify + <*> parseMaybeContributors + <*> parseJannoRemoveEmptyCols + <*> parseOnlyLatest + <*> parseForce validateOptParser :: OP.Parser ValidateOptions validateOptParser = ValidateOptions <$> parseValidatePlan diff --git a/src/Poseidon/CLI/Trident/Modify.hs b/src/Poseidon/CLI/Trident/Modify.hs new file mode 100644 index 000000000..93fa42fb0 --- /dev/null +++ b/src/Poseidon/CLI/Trident/Modify.hs @@ -0,0 +1,236 @@ +{-# LANGUAGE OverloadedStrings #-} + +module Poseidon.CLI.Trident.Modify ( + runModify, ModifyOptions (..), PackageVersionUpdate (..), ChecksumsToModify (..), + updateChecksums, addContributors, completeAndWritePackage + ) where + +import Poseidon.Core.Contributor (ContributorSpec (..)) +import Poseidon.Core.EntityTypes (HasNameAndVersion (..), + PacNameAndVersion (..), + renderNameWithVersion) +import Poseidon.Core.GenotypeData (GenotypeDataSpec (..), + GenotypeFileSpec (..)) +import Poseidon.Core.Janno (makeJannoHeader, + writeJannoFileWithoutEmptyCols) +import Poseidon.Core.Package (PackageReadOptions (..), + PoseidonPackage (..), + defaultPackageReadOptions, + readPoseidonPackageCollection, + writePoseidonPackage) +import Poseidon.Core.PoseidonVersion (PoseidonVersion (..)) +import Poseidon.Core.Utils (PoseidonIO, getChk, logDebug, + logError, logInfo, logWarning) +import Poseidon.Core.Version (VersionComponent (..), + updateThreeComponentVersion) + +import Control.DeepSeq ((<$!!>)) +import Control.Monad (when) +import Control.Monad.IO.Class (MonadIO, liftIO) +import Data.List (nub) +import Data.Maybe (fromJust) +import Data.Time (UTCTime (..), getCurrentTime) +import Data.Version (Version (..), makeVersion, + showVersion) +import System.Directory (doesFileExist, removeFile) +import System.Exit (exitFailure) +import System.FilePath (()) + +data ModifyOptions = ModifyOptions + { _modifyBaseDirs :: [FilePath] + , _modifyIgnorePoseidonVersion :: Bool + , _modifyPoseidonVersion :: Maybe Version + , _modifyPackageVersionUpdate :: Maybe PackageVersionUpdate + , _modifyChecksums :: ChecksumsToModify + , _modifyNewContributors :: Maybe [ContributorSpec] + , _modifyJannoRemoveEmptyCols :: Bool + , _modifyOnlyLatest :: Bool + , _modifyForce :: Bool + } + +data PackageVersionUpdate = PackageVersionUpdate + { _pacVerUpVersionComponent :: VersionComponent + , _pacVerUpLog :: Maybe String + } + +data ChecksumsToModify = + ChecksumNone | + ChecksumAll | + ChecksumsDetail + { _modifyChecksumGeno :: Bool + , _modifyChecksumJanno :: Bool + , _modifyChecksumSSF :: Bool + , _modifyChecksumBib :: Bool + } + +runModify :: ModifyOptions -> PoseidonIO () +runModify (ModifyOptions + baseDirs + ignorePosVer newPosVer pacVerUpdate checksumUpdate newContributors + jannoRemoveEmptyCols + onlyLatest force + ) = do + let pacReadOpts = defaultPackageReadOptions { + _readOptIgnoreChecksums = True + , _readOptIgnoreGeno = True + , _readOptGenoCheck = False + , _readOptOnlyLatest = onlyLatest + } + allPackages <- readPoseidonPackageCollection + pacReadOpts {_readOptIgnorePosVersion = ignorePosVer} + baseDirs + case allPackages of + [] -> do + logWarning "No packages found." + logInfo "Done" + [x] -> do + modifyOnePackage x + logInfo "Done" + xs -> if force + then do + mapM_ modifyOnePackage allPackages + logInfo "Done" + else do + logError $ show (length xs) ++ + " packages selected for modification.\ + \ Run modify with --force if you really want to edit\ + \ all of them." + liftIO exitFailure + where + modifyOnePackage :: PoseidonPackage -> PoseidonIO () + modifyOnePackage inPac = do + logInfo $ "Modifying package: " ++ renderNameWithVersion inPac + when jannoRemoveEmptyCols $ do + case posPacJannoFile inPac of + Nothing -> do + logWarning "No .janno file to modify with --jannoRemoveEmpty" + Just jannoPath -> do + logInfo "Reordering and removing empty columns from .janno file" + liftIO $ writeJannoFileWithoutEmptyCols + (posPacBaseDir inPac jannoPath) + (makeJannoHeader (posPacJanno inPac)) + (posPacJanno inPac) + updatedPacPosVer <- updatePoseidonVersion newPosVer inPac + updatedPacContri <- addContributors newContributors updatedPacPosVer + updatedPacChecksums <- updateChecksums checksumUpdate updatedPacContri + completeAndWritePackage pacVerUpdate updatedPacChecksums + +updatePoseidonVersion :: Maybe Version -> PoseidonPackage -> PoseidonIO PoseidonPackage +updatePoseidonVersion Nothing pac = return pac +updatePoseidonVersion (Just ver) pac = do + logDebug "Updating Poseidon version" + return pac { posPacPoseidonVersion = PoseidonVersion ver } + +addContributors :: Maybe [ContributorSpec] -> PoseidonPackage -> PoseidonIO PoseidonPackage +addContributors Nothing pac = return pac +addContributors (Just cs) pac = do + logDebug "Updating list of contributors" + return pac { posPacContributor = nub (posPacContributor pac ++ cs) } + +updateChecksums :: ChecksumsToModify -> PoseidonPackage -> PoseidonIO PoseidonPackage +updateChecksums checksumSetting pac = do + case checksumSetting of + ChecksumNone -> logDebug "Update no checksums" >> return pac + ChecksumAll -> update True True True True + ChecksumsDetail g j s b -> update g j s b + where + update :: Bool -> Bool -> Bool -> Bool -> PoseidonIO PoseidonPackage + update g j s b = do + let d = posPacBaseDir pac + let gFileSpec = genotypeFileSpec . posPacGenotypeData $ pac + newGenotypeFileSpec <- + if g + then do + logDebug "Updating genotype data checksums" + case gFileSpec of + GenotypeEigenstrat gf gfc sf sfc if_ ifc -> do + [genoChkSum, snpChkSum, indChkSum] <- + sequence [testAndGetChecksum (d f) c | (f, c) <- zip [gf, sf, if_] [gfc, sfc, ifc]] + return $ GenotypeEigenstrat gf genoChkSum sf snpChkSum if_ indChkSum + GenotypePlink gf gfc sf sfc if_ ifc -> do + [genoChkSum, snpChkSum, indChkSum] <- + sequence [testAndGetChecksum (d f) c | (f, c) <- zip [gf, sf, if_] [gfc, sfc, ifc]] + return $ GenotypePlink gf genoChkSum sf snpChkSum if_ indChkSum + GenotypeVCF gf gfc -> do + genoChkSum <- testAndGetChecksum (d gf) gfc + return $ GenotypeVCF gf genoChkSum + else return gFileSpec + newJannoChkSum <- + if j + then do + logDebug "Updating .janno file checksums" + case posPacJannoFile pac of + Nothing -> return $ posPacJannoFileChkSum pac + Just fn -> Just <$!!> getChk (d fn) + else return $ posPacJannoFileChkSum pac + newSeqSourceChkSum <- + if s + then do + logDebug "Updating .ssf file checksums" + case posPacSeqSourceFile pac of + Nothing -> return $ posPacSeqSourceFileChkSum pac + Just fn -> Just <$!!> getChk (d fn) + else return $ posPacSeqSourceFileChkSum pac + newBibChkSum <- + if b + then do + logDebug "Updating .bib file checksums" + case posPacBibFile pac of + Nothing -> return $ posPacBibFileChkSum pac + Just fn -> Just <$!!> getChk (d fn) + else return $ posPacBibFileChkSum pac + let gd = posPacGenotypeData pac + return $ pac { + posPacGenotypeData = gd {genotypeFileSpec = newGenotypeFileSpec}, + posPacJannoFileChkSum = newJannoChkSum, + posPacSeqSourceFileChkSum = newSeqSourceChkSum, + posPacBibFileChkSum = newBibChkSum + } + testAndGetChecksum :: (MonadIO m) => FilePath -> Maybe String -> m (Maybe String) + testAndGetChecksum file defaultChkSum = do + e <- liftIO . doesFileExist $ file + if e then Just <$!!> getChk file else return defaultChkSum + + +completeAndWritePackage :: Maybe PackageVersionUpdate -> PoseidonPackage -> PoseidonIO () +completeAndWritePackage Nothing pac = do + logDebug "Writing modified POSEIDON.yml file" + liftIO $ writePoseidonPackage pac +completeAndWritePackage (Just (PackageVersionUpdate component logText)) pac = do + updatedPacPacVer <- updatePackageVersion component pac + updatePacChangeLog <- writeOrUpdateChangelogFile logText updatedPacPacVer + logDebug "Writing modified POSEIDON.yml file" + liftIO $ writePoseidonPackage updatePacChangeLog + +updatePackageVersion :: VersionComponent -> PoseidonPackage -> PoseidonIO PoseidonPackage +updatePackageVersion component pac = do + logDebug "Updating package version" + (UTCTime today _) <- liftIO getCurrentTime + let pacNameAndVer = posPacNameAndVersion pac + let outPac = pac { + posPacNameAndVersion = pacNameAndVer {panavVersion = maybe (Just $ makeVersion [0, 1, 0]) + (Just . updateThreeComponentVersion component) + (getPacVersion pac) + } + , posPacLastModified = Just today + } + return outPac + +writeOrUpdateChangelogFile :: Maybe String -> PoseidonPackage -> PoseidonIO PoseidonPackage +writeOrUpdateChangelogFile Nothing pac = return pac +writeOrUpdateChangelogFile (Just logText) pac = do + case posPacChangelogFile pac of + Nothing -> do + logDebug "Creating CHANGELOG.md" + liftIO $ writeFile (posPacBaseDir pac "CHANGELOG.md") $ + "- V " ++ showVersion (fromJust $ getPacVersion pac) ++ ": " ++ + logText ++ "\n" + return pac { posPacChangelogFile = Just "CHANGELOG.md" } + Just x -> do + logDebug "Updating CHANGELOG.md" + changelogFile <- liftIO $ readFile (posPacBaseDir pac x) + liftIO $ removeFile (posPacBaseDir pac x) + liftIO $ writeFile (posPacBaseDir pac x) $ + "- V " ++ showVersion (fromJust $ getPacVersion pac) ++ ": " + ++ logText ++ "\n" ++ changelogFile + return pac diff --git a/src/Poseidon/CLI/Trident/OptparseApplicativeParsers.hs b/src/Poseidon/CLI/Trident/OptparseApplicativeParsers.hs index 387addbe9..2f28e0f2e 100644 --- a/src/Poseidon/CLI/Trident/OptparseApplicativeParsers.hs +++ b/src/Poseidon/CLI/Trident/OptparseApplicativeParsers.hs @@ -8,7 +8,7 @@ import Poseidon.CLI.Trident.Jannocoalesce (CoalesceJannoColumnSpec (.. JannoSourceSpec (..)) import Poseidon.CLI.Trident.List (ListEntity (..), RepoLocationSpec (..)) -import Poseidon.CLI.Trident.Rectify (ChecksumsToRectify (..), +import Poseidon.CLI.Trident.Modify (ChecksumsToModify (..), PackageVersionUpdate (..)) import Poseidon.CLI.Trident.Serve (ArchiveConfig (..), ArchiveSpec (..)) @@ -171,17 +171,17 @@ parseRemoveOld = OP.switch ( OP.help "Remove the old genotype files when creating the new ones." ) -parseChecksumsToRectify :: OP.Parser ChecksumsToRectify +parseChecksumsToRectify :: OP.Parser ChecksumsToModify parseChecksumsToRectify = parseChecksumNone <|> parseChecksumAll <|> parseChecksumsDetail where - parseChecksumNone :: OP.Parser ChecksumsToRectify + parseChecksumNone :: OP.Parser ChecksumsToModify parseChecksumNone = pure ChecksumNone - parseChecksumAll :: OP.Parser ChecksumsToRectify + parseChecksumAll :: OP.Parser ChecksumsToModify parseChecksumAll = ChecksumAll <$ OP.flag' () ( OP.long "checksumAll" <> OP.help "Update all checksums.") - parseChecksumsDetail :: OP.Parser ChecksumsToRectify + parseChecksumsDetail :: OP.Parser ChecksumsToModify parseChecksumsDetail = ChecksumsDetail <$> parseChecksumGeno <*> parseChecksumJanno <*> @@ -278,10 +278,8 @@ parseLog = OP.strOption ( parseForce :: OP.Parser Bool parseForce = OP.switch ( OP.long "force" <> - OP.help "Normally the POSEIDON.yml files are only changed if the \ - \poseidonVersion is adjusted or any of the checksums change. \ - \With --force a package version update can be triggered even \ - \if this is not the case." + OP.help "To prevent accidental changes to many packages, modify does not run when it is applied \ + \to more than one package. --force allows to overwrite this safeguard." ) -- this will also parse an empty list, which means "forge everything". diff --git a/src/Poseidon/CLI/Trident/Rectify.hs b/src/Poseidon/CLI/Trident/Rectify.hs index a260970fa..3f952a004 100644 --- a/src/Poseidon/CLI/Trident/Rectify.hs +++ b/src/Poseidon/CLI/Trident/Rectify.hs @@ -1,221 +1,90 @@ {-# LANGUAGE OverloadedStrings #-} module Poseidon.CLI.Trident.Rectify ( - runRectify, RectifyOptions (..), PackageVersionUpdate (..), ChecksumsToRectify (..) + runRectify, RectifyOptions (..) ) where -import Poseidon.Core.Contributor (ContributorSpec (..)) -import Poseidon.Core.EntityTypes (HasNameAndVersion (..), - PacNameAndVersion (..), - renderNameWithVersion) -import Poseidon.Core.GenotypeData (GenotypeDataSpec (..), - GenotypeFileSpec (..)) -import Poseidon.Core.Janno (makeJannoHeader, - writeJannoFileWithoutEmptyCols) -import Poseidon.Core.Package (PackageReadOptions (..), - PoseidonPackage (..), - defaultPackageReadOptions, - readPoseidonPackageCollection, - writePoseidonPackage) -import Poseidon.Core.PoseidonVersion (PoseidonVersion (..)) -import Poseidon.Core.Utils (PoseidonIO, getChecksum, - logDebug, logInfo, logWarning) -import Poseidon.Core.Version (VersionComponent (..), - updateThreeComponentVersion) +import Poseidon.CLI.Trident.Modify (ChecksumsToModify (..), + PackageVersionUpdate (..), + addContributors, + completeAndWritePackage, + updateChecksums) +import Poseidon.Core.Contributor (ContributorSpec (..)) +import Poseidon.Core.EntityTypes (renderNameWithVersion) +import Poseidon.Core.GenotypeData (GenotypeDataSpec (..), + GenotypeFileSpec (..)) +import Poseidon.Core.Package (PackageReadOptions (..), + PoseidonPackage (..), + defaultPackageReadOptions, + readPoseidonPackageCollection) +import Poseidon.Core.Utils (PoseidonIO, getChk, logInfo) -import Control.DeepSeq ((<$!!>)) -import Control.Monad (when) -import Control.Monad.IO.Class (MonadIO, liftIO) -import Data.List (nub) -import Data.Maybe (fromJust) -import Data.Time (UTCTime (..), getCurrentTime) -import Data.Version (Version (..), makeVersion, - showVersion) -import System.Directory (doesFileExist, removeFile) -import System.FilePath (()) +import Control.Monad (filterM) +import Control.Monad.IO.Class (MonadIO, liftIO) +import System.Directory (doesFileExist) +import System.FilePath (()) data RectifyOptions = RectifyOptions - { _rectifyBaseDirs :: [FilePath] - , _rectifyIgnorePoseidonVersion :: Bool - , _rectifyPoseidonVersion :: Maybe Version - , _rectifyPackageVersionUpdate :: Maybe PackageVersionUpdate - , _rectifyChecksums :: ChecksumsToRectify - , _rectifyNewContributors :: Maybe [ContributorSpec] - , _rectifyJannoRemoveEmptyCols :: Bool - , _rectifyOnlyLatest :: Bool - } - -data PackageVersionUpdate = PackageVersionUpdate - { _pacVerUpVersionComponent :: VersionComponent - , _pacVerUpLog :: Maybe String - } - -data ChecksumsToRectify = - ChecksumNone | - ChecksumAll | - ChecksumsDetail - { _rectifyChecksumGeno :: Bool - , _rectifyChecksumJanno :: Bool - , _rectifyChecksumSSF :: Bool - , _rectifyChecksumBib :: Bool + { _rectifyBaseDirs :: [FilePath] + , _rectifyPackageVersionUpdate :: Maybe PackageVersionUpdate + , _rectifyNewContributors :: Maybe [ContributorSpec] } runRectify :: RectifyOptions -> PoseidonIO () -runRectify (RectifyOptions - baseDirs - ignorePosVer newPosVer pacVerUpdate checksumUpdate newContributors - jannoRemoveEmptyCols - onlyLatest - ) = do +runRectify (RectifyOptions baseDirs pacVerUpdate newContributors) = do let pacReadOpts = defaultPackageReadOptions { _readOptIgnoreChecksums = True , _readOptIgnoreGeno = True , _readOptGenoCheck = False - , _readOptOnlyLatest = onlyLatest + , _readOptOnlyLatest = False + , _readOptIgnorePosVersion = True } - allPackages <- readPoseidonPackageCollection - pacReadOpts {_readOptIgnorePosVersion = ignorePosVer} - baseDirs - logInfo "Starting per-package update procedure" - mapM_ rectifyOnePackage allPackages + allPackages <- readPoseidonPackageCollection pacReadOpts baseDirs + logInfo "Searching packages that need rectification" + toRectifyPackages <- filterM needsRectification allPackages + case toRectifyPackages of + [] -> logInfo "No packages need rectification" + xs -> do + logInfo $ show (length xs) ++ " packages need rectification" + mapM_ rectifyOnePackage xs logInfo "Done" where rectifyOnePackage :: PoseidonPackage -> PoseidonIO () - rectifyOnePackage inPac = do - logInfo $ "Rectifying package: " ++ renderNameWithVersion inPac - when jannoRemoveEmptyCols $ do - case posPacJannoFile inPac of - Nothing -> do - logWarning "No .janno file to modify with --jannoRemoveEmpty" - Just jannoPath -> do - logInfo "Reordering and removing empty columns from .janno file" - liftIO $ writeJannoFileWithoutEmptyCols - (posPacBaseDir inPac jannoPath) - (makeJannoHeader (posPacJanno inPac)) - (posPacJanno inPac) - updatedPacPosVer <- updatePoseidonVersion newPosVer inPac - updatedPacContri <- addContributors newContributors updatedPacPosVer - updatedPacChecksums <- updateChecksums checksumUpdate updatedPacContri - completeAndWritePackage pacVerUpdate updatedPacChecksums - -updatePoseidonVersion :: Maybe Version -> PoseidonPackage -> PoseidonIO PoseidonPackage -updatePoseidonVersion Nothing pac = return pac -updatePoseidonVersion (Just ver) pac = do - logDebug "Updating Poseidon version" - return pac { posPacPoseidonVersion = PoseidonVersion ver } - -addContributors :: Maybe [ContributorSpec] -> PoseidonPackage -> PoseidonIO PoseidonPackage -addContributors Nothing pac = return pac -addContributors (Just cs) pac = do - logDebug "Updating list of contributors" - return pac { posPacContributor = nub (posPacContributor pac ++ cs) } - -updateChecksums :: ChecksumsToRectify -> PoseidonPackage -> PoseidonIO PoseidonPackage -updateChecksums checksumSetting pac = do - case checksumSetting of - ChecksumNone -> logDebug "Update no checksums" >> return pac - ChecksumAll -> update True True True True - ChecksumsDetail g j s b -> update g j s b - where - update :: Bool -> Bool -> Bool -> Bool -> PoseidonIO PoseidonPackage - update g j s b = do - let d = posPacBaseDir pac - let gFileSpec = genotypeFileSpec . posPacGenotypeData $ pac - newGenotypeFileSpec <- - if g - then do - logDebug "Updating genotype data checksums" - case gFileSpec of - GenotypeEigenstrat gf gfc sf sfc if_ ifc -> do - [genoChkSum, snpChkSum, indChkSum] <- - sequence [testAndGetChecksum (d f) c | (f, c) <- zip [gf, sf, if_] [gfc, sfc, ifc]] - return $ GenotypeEigenstrat gf genoChkSum sf snpChkSum if_ indChkSum - GenotypePlink gf gfc sf sfc if_ ifc -> do - [genoChkSum, snpChkSum, indChkSum] <- - sequence [testAndGetChecksum (d f) c | (f, c) <- zip [gf, sf, if_] [gfc, sfc, ifc]] - return $ GenotypePlink gf genoChkSum sf snpChkSum if_ indChkSum - GenotypeVCF gf gfc -> do - genoChkSum <- testAndGetChecksum (d gf) gfc - return $ GenotypeVCF gf genoChkSum - else return gFileSpec - newJannoChkSum <- - if j - then do - logDebug "Updating .janno file checksums" - case posPacJannoFile pac of - Nothing -> return $ posPacJannoFileChkSum pac - Just fn -> Just <$!!> getChk (d fn) - else return $ posPacJannoFileChkSum pac - newSeqSourceChkSum <- - if s - then do - logDebug "Updating .ssf file checksums" - case posPacSeqSourceFile pac of - Nothing -> return $ posPacSeqSourceFileChkSum pac - Just fn -> Just <$!!> getChk (d fn) - else return $ posPacSeqSourceFileChkSum pac - newBibChkSum <- - if b - then do - logDebug "Updating .bib file checksums" - case posPacBibFile pac of - Nothing -> return $ posPacBibFileChkSum pac - Just fn -> Just <$!!> getChk (d fn) - else return $ posPacBibFileChkSum pac - let gd = posPacGenotypeData pac - return $ pac { - posPacGenotypeData = gd {genotypeFileSpec = newGenotypeFileSpec}, - posPacJannoFileChkSum = newJannoChkSum, - posPacSeqSourceFileChkSum = newSeqSourceChkSum, - posPacBibFileChkSum = newBibChkSum - } - getChk :: (MonadIO m) => FilePath -> m String - getChk = liftIO . getChecksum - testAndGetChecksum :: (MonadIO m) => FilePath -> Maybe String -> m (Maybe String) - testAndGetChecksum file defaultChkSum = do - e <- liftIO . doesFileExist $ file - if e then Just <$!!> getChk file else return defaultChkSum - - -completeAndWritePackage :: Maybe PackageVersionUpdate -> PoseidonPackage -> PoseidonIO () -completeAndWritePackage Nothing pac = do - logDebug "Writing rectified POSEIDON.yml file" - liftIO $ writePoseidonPackage pac -completeAndWritePackage (Just (PackageVersionUpdate component logText)) pac = do - updatedPacPacVer <- updatePackageVersion component pac - updatePacChangeLog <- writeOrUpdateChangelogFile logText updatedPacPacVer - logDebug "Writing rectified POSEIDON.yml file" - liftIO $ writePoseidonPackage updatePacChangeLog + rectifyOnePackage pac = do + logInfo $ "Rectifying package: " ++ renderNameWithVersion pac + pure pac >>= + updateChecksums ChecksumAll >>= + addContributors newContributors >>= + completeAndWritePackage pacVerUpdate -updatePackageVersion :: VersionComponent -> PoseidonPackage -> PoseidonIO PoseidonPackage -updatePackageVersion component pac = do - logDebug "Updating package version" - (UTCTime today _) <- liftIO getCurrentTime - let pacNameAndVer = posPacNameAndVersion pac - let outPac = pac { - posPacNameAndVersion = pacNameAndVer {panavVersion = maybe (Just $ makeVersion [0, 1, 0]) - (Just . updateThreeComponentVersion component) - (getPacVersion pac) - } - , posPacLastModified = Just today - } - return outPac +needsRectification :: PoseidonPackage -> PoseidonIO Bool +needsRectification pac = do + let d = posPacBaseDir pac + gFileSpec = genotypeFileSpec . posPacGenotypeData $ pac + goodGeno <- case gFileSpec of + GenotypeEigenstrat gf gfc sf sfc if_ ifc -> + and <$> sequence [goodChecksum d (Just f) c | (f, c) <- zip [gf, sf, if_] [gfc, sfc, ifc]] + GenotypePlink gf gfc sf sfc if_ ifc -> + and <$> sequence [goodChecksum d (Just f) c | (f, c) <- zip [gf, sf, if_] [gfc, sfc, ifc]] + GenotypeVCF gf gfc -> goodChecksum d (Just gf) gfc + goodJanno <- goodChecksum d (posPacJannoFile pac) (posPacJannoFileChkSum pac) + goodSeqSo <- goodChecksum d (posPacSeqSourceFile pac) (posPacSeqSourceFileChkSum pac) + goodBib <- goodChecksum d (posPacBibFile pac) (posPacBibFileChkSum pac) + let needsRect = not $ and [goodGeno, goodJanno, goodSeqSo, goodBib] + logInfo $ (if needsRect then "CHANGED " else "OK ") ++ renderNameWithVersion pac + return needsRect -writeOrUpdateChangelogFile :: Maybe String -> PoseidonPackage -> PoseidonIO PoseidonPackage -writeOrUpdateChangelogFile Nothing pac = return pac -writeOrUpdateChangelogFile (Just logText) pac = do - case posPacChangelogFile pac of - Nothing -> do - logDebug "Creating CHANGELOG.md" - liftIO $ writeFile (posPacBaseDir pac "CHANGELOG.md") $ - "- V " ++ showVersion (fromJust $ getPacVersion pac) ++ ": " ++ - logText ++ "\n" - return pac { posPacChangelogFile = Just "CHANGELOG.md" } - Just x -> do - logDebug "Updating CHANGELOG.md" - changelogFile <- liftIO $ readFile (posPacBaseDir pac x) - liftIO $ removeFile (posPacBaseDir pac x) - liftIO $ writeFile (posPacBaseDir pac x) $ - "- V " ++ showVersion (fromJust $ getPacVersion pac) ++ ": " - ++ logText ++ "\n" ++ changelogFile - return pac +goodChecksum :: (MonadIO m) => FilePath -> Maybe FilePath -> Maybe String -> m Bool +-- no file: ok +goodChecksum _ Nothing _ = return True +-- no checksum: not ok +goodChecksum _ _ Nothing = return False +-- file and checksum: is the checksum correct? +goodChecksum baseDir (Just file) (Just expectedCheckSum) = do + let f = baseDir file + exists <- liftIO . doesFileExist $ f + if exists + then do + realChecksum <- getChk f + return $ realChecksum == expectedCheckSum + else return True diff --git a/src/Poseidon/Core/Package.hs b/src/Poseidon/Core/Package.hs index 467051c4a..0c9afed2f 100644 --- a/src/Poseidon/Core/Package.hs +++ b/src/Poseidon/Core/Package.hs @@ -11,6 +11,7 @@ module Poseidon.Core.Package ( findAllPoseidonYmlFiles, checkJannoIndConsistency, checkGenoFiles, + readPoseidonYaml, readPoseidonPackageCollection, readPoseidonPackageCollectionWithSkipIndicator, getJointGenotypeData, @@ -431,6 +432,13 @@ readPoseidonPackageCollectionWithSkipIndicator opts baseDirs = do logDebug $ "Package " ++ show numberPackage ++ ": " ++ path try . readPoseidonPackage opts $ path +readPoseidonYaml :: FilePath -> IO PoseidonYamlStruct +readPoseidonYaml path = do + bs <- liftIO $ B.readFile path + case decodeEither' bs of + Left err -> throwM $ PoseidonYamlParseException path err + Right x -> return x + -- | A function to read in a poseidon package from a YAML file. Note that this function calls the addFullPaths function to -- make paths absolute. readPoseidonPackage :: PackageReadOptions @@ -438,12 +446,10 @@ readPoseidonPackage :: PackageReadOptions -> PoseidonIO PoseidonPackage -- ^ the returning package returned in the IO monad. readPoseidonPackage opts ymlPath = do let baseDir = takeDirectory ymlPath - bs <- liftIO $ B.readFile ymlPath -- read yml files - yml@(PoseidonYamlStruct ver tit des con pacVer mod_ lic geno jannoF jannoC seqSourceF seqSourceC bibF bibC readF changeF) <- case decodeEither' bs of - Left err -> throwM $ PoseidonYamlParseException ymlPath err - Right pac -> return pac + yml@(PoseidonYamlStruct ver tit des con pacVer mod_ lic geno jannoF jannoC seqSourceF seqSourceC bibF bibC readF changeF) <- + liftIO $ readPoseidonYaml ymlPath checkYML yml -- file existence and checksum test diff --git a/src/Poseidon/Core/Utils.hs b/src/Poseidon/Core/Utils.hs index 81e0e1d50..1a4a1fb3c 100644 --- a/src/Poseidon/Core/Utils.hs +++ b/src/Poseidon/Core/Utils.hs @@ -14,7 +14,7 @@ module Poseidon.Core.Utils ( checkFile, checkLineEnding, checkLineEndingIfNotZipped, - getChecksum, + getChecksum, getChk, logWarning, logInfo, logDebug, @@ -38,15 +38,17 @@ import Colog (HasLog (..), LogAction (..), Message, Msg (..), Severity (..), cfilter, cmapM, logTextStderr, msgSeverity, msgText, showSeverity) -import Control.Exception (Exception (..), throwIO) +import Control.Exception (Exception (..), evaluate, throwIO) import Control.Exception.Base (SomeException) import Control.Monad (unless, when) import Control.Monad.Catch (throwM) import Control.Monad.IO.Class (MonadIO, liftIO) import Control.Monad.Reader (ReaderT, asks, runReaderT) +import Crypto.Hash.MD5 as MD5 import qualified Data.ByteString as BS -import qualified Data.ByteString.Lazy as LB -import Data.Digest.Pure.MD5 (md5) +import Data.ByteString.Base16 as B16 +import qualified Data.ByteString.Char8 as B8 +import qualified Data.ByteString.Lazy as BL import Data.List (isSuffixOf) import qualified Data.Set as Set import Data.Text (Text, pack) @@ -297,11 +299,14 @@ checkFile fn maybeChkSum = do when (fnChkSum /= chkSum) $ throwM (PoseidonFileChecksumException fn) -- helper functions to get the checksum of a file +getChk :: (MonadIO m) => FilePath -> m String +getChk = liftIO . getChecksum getChecksum :: FilePath -> IO String -getChecksum f = do - fileContent <- LB.readFile f - let md5Digest = md5 fileContent - return $ show md5Digest +getChecksum f = + withBinaryFile f ReadMode $ \h -> do + contents <- BL.hGetContents h + digest <- evaluate (MD5.hashlazy contents) + pure $ B8.unpack $ B16.encode digest -- helper function to check line endings of text files -- only considers the first line: diff --git a/test/PoseidonGoldenTests/GoldenTestCheckSumFile.txt b/test/PoseidonGoldenTests/GoldenTestCheckSumFile.txt index 5c3b1c59a..209b1c8dd 100644 --- a/test/PoseidonGoldenTests/GoldenTestCheckSumFile.txt +++ b/test/PoseidonGoldenTests/GoldenTestCheckSumFile.txt @@ -58,15 +58,21 @@ a9fcb59cb933d8f183f3c3f6fbf2e213 genoconvert init_vcf/Schiffels_vcf/geno.fam 3c38f40efe215a047c02f4e98e0390da genoconvert genoconvert/zip_roundtrip/Schiffels_2016.bed 8538ffd971ebb12cf5ef6e338da27970 genoconvert genoconvert/zip_roundtrip/Schiffels_2016.bim a9fcb59cb933d8f183f3c3f6fbf2e213 genoconvert genoconvert/zip_roundtrip/Schiffels_2016.fam -57c109be74d7c596bcb76aae03b2d124 rectify init/Schiffels/POSEIDON.yml -4aeae97cfc44b55b2a8005425d786148 rectify init/Schiffels/CHANGELOG.md -174375c685f2e1b7432172e6075f1f2f rectify init/Schiffels/POSEIDON.yml -3bb396e099d5b8771a3409f5fe85d70b rectify init/Schiffels/CHANGELOG.md -667074fbd38a002cf44750e92346a118 rectify init/Schiffels/POSEIDON.yml -3bb396e099d5b8771a3409f5fe85d70b rectify init/Schiffels/CHANGELOG.md -ebf0b456d5e09686ee7131db06848104 rectify init/Schiffels/POSEIDON.yml -3bb396e099d5b8771a3409f5fe85d70b rectify init/Schiffels/CHANGELOG.md -083fe7ef4206c979356a3a2454d780b1 rectify init/Schiffels/Schiffels.janno +f63b7186e18c1c6935312a3ab2b0ac6a rectify rectify/Schmid_2028/POSEIDON.yml +53f3a0f7f8afc56a452c831111fe9f0d rectify rectify/Schmid_2028/CHANGELOG.md +581fac2d8df63a48cd00fe0a80cfaba8 rectify rectify/Schiffels_2016/POSEIDON.yml +ffe2b8832ad21fbd44faff1f0b9075de rectify rectify/Schiffels_2016/CHANGELOG.md +2e0fe6d764926dff57f2fa421bf295c9 rectify rectify/Schiffels_2016/sources.bib +e234a2511ce81ff2002b670c5d931b94 rectify rectify/Wang_2020/POSEIDON.yml +57c109be74d7c596bcb76aae03b2d124 modify init/Schiffels/POSEIDON.yml +4aeae97cfc44b55b2a8005425d786148 modify init/Schiffels/CHANGELOG.md +174375c685f2e1b7432172e6075f1f2f modify init/Schiffels/POSEIDON.yml +3bb396e099d5b8771a3409f5fe85d70b modify init/Schiffels/CHANGELOG.md +667074fbd38a002cf44750e92346a118 modify init/Schiffels/POSEIDON.yml +3bb396e099d5b8771a3409f5fe85d70b modify init/Schiffels/CHANGELOG.md +ebf0b456d5e09686ee7131db06848104 modify init/Schiffels/POSEIDON.yml +3bb396e099d5b8771a3409f5fe85d70b modify init/Schiffels/CHANGELOG.md +083fe7ef4206c979356a3a2454d780b1 modify init/Schiffels/Schiffels.janno 5b5c20071f3e53346fdfa04f9dbb0223 forge forge/ForgePac1/POSEIDON.yml 1286a2580e4bfbed7d804d5f3fe125f7 forge forge/ForgePac1/ForgePac1.geno 848cf0fab32e4078a88cc17ad8ec3381 forge forge/ForgePac1/ForgePac1.janno @@ -154,7 +160,7 @@ ebf0b456d5e09686ee7131db06848104 timetravel timetravel/Schiffels-1.1.1/POSEIDON. ea9d00246c9064ea8f1f35d4b30acb49 fetch fetch/by_package/Schmid_2028-1.0.0/Schmid_2028.janno 70547a0bdb20d07dc8e708a1a8f055e4 fetch fetch/by_package/Schmid_2028-1.0.0/geno.txt b43da4d5734371c0648553120f812466 fetch fetch/by_package/Lamnidis_2018-1.0.0/POSEIDON.yml -32c0c39ad05a40c13828e92cd2062a46 fetch fetch/by_individual/Wang_2020-0.1.0/POSEIDON.yml +e234a2511ce81ff2002b670c5d931b94 fetch fetch/by_individual/Wang_2020-0.1.0/POSEIDON.yml 1ab24c45ef3a13e0fb34afac7a21dca8 fetch fetch/by_individual/Schmid_2028-1.0.0/POSEIDON.yml b43da4d5734371c0648553120f812466 fetch fetch/by_individual/Lamnidis_2018-1.0.0/POSEIDON.yml b43da4d5734371c0648553120f812466 fetch fetch/multi_packages_1/Lamnidis_2018-1.0.0/POSEIDON.yml diff --git a/test/PoseidonGoldenTests/GoldenTestData/chronicle/Wang_2020/POSEIDON.yml b/test/PoseidonGoldenTests/GoldenTestData/chronicle/Wang_2020/POSEIDON.yml index 1e137405f..e7ff75a1a 100644 --- a/test/PoseidonGoldenTests/GoldenTestData/chronicle/Wang_2020/POSEIDON.yml +++ b/test/PoseidonGoldenTests/GoldenTestData/chronicle/Wang_2020/POSEIDON.yml @@ -2,16 +2,22 @@ poseidonVersion: 2.7.1 title: Wang_2020 description: Genetic data published in Wang et al. 2020, Plink test contributor: - - name: Ke Wang - email: wang@institute.org +- name: Ke Wang + email: wang@institute.org packageVersion: 0.1.0 lastModified: 2020-05-20 -bibFile: sources.bib genotypeData: format: PLINK genoFile: Wang_2020.bed + genoFileChkSum: ae66d851301f4a761b819f97ec28fa55 snpFile: Wang_2020.bim + snpFileChkSum: f96ea12084830fb43b3a24fd867f0ba7 indFile: Wang_2020.fam + indFileChkSum: 575ea54b293e297b5f354c910fe1610d snpSet: Other jannoFile: Wang_2020.janno -sequencingSourceFile: Wang_2020.ssf \ No newline at end of file +jannoFileChkSum: d271413712cf63767fc12dc3761de2e4 +sequencingSourceFile: Wang_2020.ssf +sequencingSourceFileChkSum: 76f24988a5440fea474308f9151548a3 +bibFile: sources.bib +bibFileChkSum: 67a9340931939f6d49fefe132c6ae984 diff --git a/test/PoseidonGoldenTests/GoldenTestData/fetch/by_individual/Wang_2020-0.1.0/POSEIDON.yml b/test/PoseidonGoldenTests/GoldenTestData/fetch/by_individual/Wang_2020-0.1.0/POSEIDON.yml index 1e137405f..e7ff75a1a 100644 --- a/test/PoseidonGoldenTests/GoldenTestData/fetch/by_individual/Wang_2020-0.1.0/POSEIDON.yml +++ b/test/PoseidonGoldenTests/GoldenTestData/fetch/by_individual/Wang_2020-0.1.0/POSEIDON.yml @@ -2,16 +2,22 @@ poseidonVersion: 2.7.1 title: Wang_2020 description: Genetic data published in Wang et al. 2020, Plink test contributor: - - name: Ke Wang - email: wang@institute.org +- name: Ke Wang + email: wang@institute.org packageVersion: 0.1.0 lastModified: 2020-05-20 -bibFile: sources.bib genotypeData: format: PLINK genoFile: Wang_2020.bed + genoFileChkSum: ae66d851301f4a761b819f97ec28fa55 snpFile: Wang_2020.bim + snpFileChkSum: f96ea12084830fb43b3a24fd867f0ba7 indFile: Wang_2020.fam + indFileChkSum: 575ea54b293e297b5f354c910fe1610d snpSet: Other jannoFile: Wang_2020.janno -sequencingSourceFile: Wang_2020.ssf \ No newline at end of file +jannoFileChkSum: d271413712cf63767fc12dc3761de2e4 +sequencingSourceFile: Wang_2020.ssf +sequencingSourceFileChkSum: 76f24988a5440fea474308f9151548a3 +bibFile: sources.bib +bibFileChkSum: 67a9340931939f6d49fefe132c6ae984 diff --git a/test/PoseidonGoldenTests/GoldenTestData/genoconvert/out_vcf/Schiffels_2016.vcf.gz b/test/PoseidonGoldenTests/GoldenTestData/genoconvert/out_vcf/Schiffels_2016.vcf.gz index 8de0bfebe..9b34a0769 100644 Binary files a/test/PoseidonGoldenTests/GoldenTestData/genoconvert/out_vcf/Schiffels_2016.vcf.gz and b/test/PoseidonGoldenTests/GoldenTestData/genoconvert/out_vcf/Schiffels_2016.vcf.gz differ diff --git a/test/PoseidonGoldenTests/GoldenTestData/rectify/Schiffels_2016/CHANGELOG.md b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schiffels_2016/CHANGELOG.md new file mode 100644 index 000000000..bbaa5b489 --- /dev/null +++ b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schiffels_2016/CHANGELOG.md @@ -0,0 +1,2 @@ +- V 2.0.0: rectify test +V 1.0.1: not specified diff --git a/test/PoseidonGoldenTests/GoldenTestData/rectify/Schiffels_2016/POSEIDON.yml b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schiffels_2016/POSEIDON.yml new file mode 100644 index 000000000..cd64ad521 --- /dev/null +++ b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schiffels_2016/POSEIDON.yml @@ -0,0 +1,32 @@ +poseidonVersion: 2.7.1 +title: Schiffels_2016 +description: Genetic data published in Schiffels et al. 2016 +contributor: +- name: Stephan Schiffels + email: schiffels@institute.org +- name: Josiah Carberry + email: carberry@brown.edu + orcid: 0000-0002-1825-0097 +- name: rectify + email: rectify@example.org +packageVersion: 2.0.0 +lastModified: 1970-01-01 +genotypeData: + format: EIGENSTRAT + genoFile: geno.txt + genoFileChkSum: 0332344057c0c4dce2ff7176f8e1103d + snpFile: snp.txt + snpFileChkSum: d76e3e7a8fc0f1f5e435395424b5aeab + indFile: ind.txt + indFileChkSum: f77dc756666dbfef3bb35191ae15a167 + snpSet: Other + referenceGenomeAssembly: GRCh37 + referenceGenomeAssemblyURL: https://www.ncbi.nlm.nih.gov/datasets/genome/GCA_000001405.14 +jannoFile: Schiffels_2016.janno +jannoFileChkSum: 09e65688bbb0d315648ccc7de0bf03e8 +sequencingSourceFile: ena_table.ssf +sequencingSourceFileChkSum: 05c2e7c8163bd2ed81ca888696c00cb1 +bibFile: sources.bib +bibFileChkSum: 2e0fe6d764926dff57f2fa421bf295c9 +readmeFile: README.md +changelogFile: CHANGELOG.md diff --git a/test/PoseidonGoldenTests/GoldenTestData/rectify/Schiffels_2016/README.md b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schiffels_2016/README.md new file mode 100644 index 000000000..af27ff498 --- /dev/null +++ b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schiffels_2016/README.md @@ -0,0 +1 @@ +This is a test file. \ No newline at end of file diff --git a/test/PoseidonGoldenTests/GoldenTestData/rectify/Schiffels_2016/Schiffels_2016.janno b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schiffels_2016/Schiffels_2016.janno new file mode 100755 index 000000000..69d7e0cd7 --- /dev/null +++ b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schiffels_2016/Schiffels_2016.janno @@ -0,0 +1,11 @@ +Poseidon_ID Group_Name Genetic_Sex Publication AddCol1 AddCol2 +XXX001 POP1 M Schiffels2016 v1 v2 +XXX002 POP2 F Schiffels2016 v1 v2 +XXX003 POP1 M Schiffels2016 v1 v2 +XXX004 POP2 F Schiffels2016 v1 v2 +XXX005 POP2 M Schiffels2016;TestPaper1 v1 v2 +XXX006 POP2 F Schiffels2016;TestPaper1 v1 v2 +XXX007 POP1 M Schiffels2016;TestBook1 v1 v2 +XXX008 POP3 F Schiffels2016;TestBook1 v1 v2 +XXX009 POP1 F Schiffels2016;TestPaper1;TestBook1 v1 v2 +XXX010 POP3 M Schiffels2016;TestPaper1;TestBook1 v1 v2 diff --git a/test/PoseidonGoldenTests/GoldenTestData/rectify/Schiffels_2016/ena_table.ssf b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schiffels_2016/ena_table.ssf new file mode 100644 index 000000000..4bde27b8f --- /dev/null +++ b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schiffels_2016/ena_table.ssf @@ -0,0 +1,5 @@ +poseidon_IDs run_accession other_info_1 other_info_2 +XXX001;XXX002 ERR3518150 A B +XXX002;XXX004;XXX005 ERR3518151 C D +XXX003 ERR3518152 E F +XXX001 ERR3518153 G H diff --git a/test/PoseidonGoldenTests/GoldenTestData/rectify/Schiffels_2016/geno.txt b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schiffels_2016/geno.txt new file mode 100644 index 000000000..e001a769b --- /dev/null +++ b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schiffels_2016/geno.txt @@ -0,0 +1,9 @@ +2000000100 +2022221222 +0000910000 +0110100000 +1121901201 +2222222222 +2292221221 +2201221220 +2292122212 diff --git a/test/PoseidonGoldenTests/GoldenTestData/rectify/Schiffels_2016/ind.txt b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schiffels_2016/ind.txt new file mode 100644 index 000000000..4dd633867 --- /dev/null +++ b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schiffels_2016/ind.txt @@ -0,0 +1,10 @@ +XXX001 M POP1 +XXX002 F POP2 +XXX003 M POP1 +XXX004 F POP2 +XXX005 M POP2 +XXX006 F POP2 +XXX007 M POP1 +XXX008 F POP3 +XXX009 F POP1 +XXX010 M POP3 diff --git a/test/PoseidonGoldenTests/GoldenTestData/rectify/Schiffels_2016/snp.txt b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schiffels_2016/snp.txt new file mode 100644 index 000000000..6a56b2435 --- /dev/null +++ b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schiffels_2016/snp.txt @@ -0,0 +1,9 @@ +1_752566 1 0.020130 752566 G A +1_842013 1 0.022518 842013 T G +1_891021 1 0.024116 891021 G A +1_949654 1 0.025727 949654 A G +2_1018704 2 0.026288 1018704 A G +2_1045331 2 0.026665 1045331 G A +2_1048955 2 0.026674 1048955 A G +2_1061166 2 0.026711 1061166 T C +2_1108637 2 0.028311 1108637 G A diff --git a/test/PoseidonGoldenTests/GoldenTestData/rectify/Schiffels_2016/sources.bib b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schiffels_2016/sources.bib new file mode 100644 index 000000000..f5cbdd998 --- /dev/null +++ b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schiffels_2016/sources.bib @@ -0,0 +1,12 @@ +@article{Schiffels2016, + title = Test +} + +@article{TestPaper1, + title = TestPaper +} + +@book{TestBook1, + title = TestBook +} + diff --git a/test/PoseidonGoldenTests/GoldenTestData/rectify/Schmid_2028/CHANGELOG.md b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schmid_2028/CHANGELOG.md new file mode 100644 index 000000000..0989962ea --- /dev/null +++ b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schmid_2028/CHANGELOG.md @@ -0,0 +1 @@ +- V 2.0.0: rectify test diff --git a/test/PoseidonGoldenTests/GoldenTestData/rectify/Schmid_2028/POSEIDON.yml b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schmid_2028/POSEIDON.yml new file mode 100644 index 000000000..9cc63f9f1 --- /dev/null +++ b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schmid_2028/POSEIDON.yml @@ -0,0 +1,22 @@ +poseidonVersion: 2.6.0 +title: Schmid_2028 +description: Genetic data that will never be published in Schmid et al. 2028 +contributor: +- name: Clemens Schmid + email: schmid@institute.org +- name: rectify + email: rectify@example.org +packageVersion: 2.0.0 +lastModified: 1970-01-01 +genotypeData: + format: EIGENSTRAT + genoFile: geno.txt + genoFileChkSum: 70547a0bdb20d07dc8e708a1a8f055e4 + snpFile: snp.txt + snpFileChkSum: d76e3e7a8fc0f1f5e435395424b5aeab + indFile: ind.txt + indFileChkSum: f335dfd53eb4db511b08dcaccc81505b + snpSet: Other +jannoFile: Schmid_2028.janno +jannoFileChkSum: ea9d00246c9064ea8f1f35d4b30acb49 +changelogFile: CHANGELOG.md diff --git a/test/PoseidonGoldenTests/GoldenTestData/rectify/Schmid_2028/Schmid_2028.janno b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schmid_2028/Schmid_2028.janno new file mode 100755 index 000000000..108642f94 --- /dev/null +++ b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schmid_2028/Schmid_2028.janno @@ -0,0 +1,3 @@ +Poseidon_ID Group_Name Genetic_Sex +XXX001 POP1 F +XXX002 POP2 F diff --git a/test/PoseidonGoldenTests/GoldenTestData/rectify/Schmid_2028/geno.txt b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schmid_2028/geno.txt new file mode 100644 index 000000000..e5e735c61 --- /dev/null +++ b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schmid_2028/geno.txt @@ -0,0 +1,9 @@ +00 +00 +00 +00 +00 +00 +00 +00 +00 diff --git a/test/PoseidonGoldenTests/GoldenTestData/rectify/Schmid_2028/ind.txt b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schmid_2028/ind.txt new file mode 100644 index 000000000..5f99068fe --- /dev/null +++ b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schmid_2028/ind.txt @@ -0,0 +1,2 @@ +XXX001 F POP1 +XXX002 F POP2 diff --git a/test/PoseidonGoldenTests/GoldenTestData/rectify/Schmid_2028/snp.txt b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schmid_2028/snp.txt new file mode 100644 index 000000000..6a56b2435 --- /dev/null +++ b/test/PoseidonGoldenTests/GoldenTestData/rectify/Schmid_2028/snp.txt @@ -0,0 +1,9 @@ +1_752566 1 0.020130 752566 G A +1_842013 1 0.022518 842013 T G +1_891021 1 0.024116 891021 G A +1_949654 1 0.025727 949654 A G +2_1018704 2 0.026288 1018704 A G +2_1045331 2 0.026665 1045331 G A +2_1048955 2 0.026674 1048955 A G +2_1061166 2 0.026711 1061166 T C +2_1108637 2 0.028311 1108637 G A diff --git a/test/PoseidonGoldenTests/GoldenTestData/rectify/Wang_2020/POSEIDON.yml b/test/PoseidonGoldenTests/GoldenTestData/rectify/Wang_2020/POSEIDON.yml new file mode 100644 index 000000000..e7ff75a1a --- /dev/null +++ b/test/PoseidonGoldenTests/GoldenTestData/rectify/Wang_2020/POSEIDON.yml @@ -0,0 +1,23 @@ +poseidonVersion: 2.7.1 +title: Wang_2020 +description: Genetic data published in Wang et al. 2020, Plink test +contributor: +- name: Ke Wang + email: wang@institute.org +packageVersion: 0.1.0 +lastModified: 2020-05-20 +genotypeData: + format: PLINK + genoFile: Wang_2020.bed + genoFileChkSum: ae66d851301f4a761b819f97ec28fa55 + snpFile: Wang_2020.bim + snpFileChkSum: f96ea12084830fb43b3a24fd867f0ba7 + indFile: Wang_2020.fam + indFileChkSum: 575ea54b293e297b5f354c910fe1610d + snpSet: Other +jannoFile: Wang_2020.janno +jannoFileChkSum: d271413712cf63767fc12dc3761de2e4 +sequencingSourceFile: Wang_2020.ssf +sequencingSourceFileChkSum: 76f24988a5440fea474308f9151548a3 +bibFile: sources.bib +bibFileChkSum: 67a9340931939f6d49fefe132c6ae984 diff --git a/test/PoseidonGoldenTests/GoldenTestData/rectify/Wang_2020/Wang_2020.bed b/test/PoseidonGoldenTests/GoldenTestData/rectify/Wang_2020/Wang_2020.bed new file mode 100644 index 000000000..b75a53563 --- /dev/null +++ b/test/PoseidonGoldenTests/GoldenTestData/rectify/Wang_2020/Wang_2020.bed @@ -0,0 +1 @@ +lê«‹¨èª/¨è«¯ª « \ No newline at end of file diff --git a/test/PoseidonGoldenTests/GoldenTestData/rectify/Wang_2020/Wang_2020.bim b/test/PoseidonGoldenTests/GoldenTestData/rectify/Wang_2020/Wang_2020.bim new file mode 100644 index 000000000..d6fa8180c --- /dev/null +++ b/test/PoseidonGoldenTests/GoldenTestData/rectify/Wang_2020/Wang_2020.bim @@ -0,0 +1,7 @@ +11 rs0000 0.000000 0 A C +11 rs1111 0.001000 100000 A G +11 rs2222 0.002000 200000 A T +11 rs3333 0.003000 300000 C A +11 rs4444 0.004000 400000 G A +11 rs5555 0.005000 500000 T A +11 rs6666 0.006000 600000 G T diff --git a/test/PoseidonGoldenTests/GoldenTestData/rectify/Wang_2020/Wang_2020.fam b/test/PoseidonGoldenTests/GoldenTestData/rectify/Wang_2020/Wang_2020.fam new file mode 100644 index 000000000..a8e0dbb72 --- /dev/null +++ b/test/PoseidonGoldenTests/GoldenTestData/rectify/Wang_2020/Wang_2020.fam @@ -0,0 +1,5 @@ + 1 SAMPLE0 0 0 2 2 + 2 SAMPLE1 0 0 1 2 + 3 SAMPLE2 0 0 2 1 + 4 SAMPLE3 0 0 1 1 + 5 SAMPLE4 0 0 2 1 diff --git a/test/PoseidonGoldenTests/GoldenTestData/rectify/Wang_2020/Wang_2020.janno b/test/PoseidonGoldenTests/GoldenTestData/rectify/Wang_2020/Wang_2020.janno new file mode 100755 index 000000000..28ef9e1e1 --- /dev/null +++ b/test/PoseidonGoldenTests/GoldenTestData/rectify/Wang_2020/Wang_2020.janno @@ -0,0 +1,6 @@ +Poseidon_ID Group_Name Genetic_Sex Publication +SAMPLE0 1 F n/a +SAMPLE1 2 M TestPaper1 +SAMPLE2 3 F Wang2020;TestPaper1 +SAMPLE3 4 M Wang2020;TestBook2 +SAMPLE4 5 F Wang2020;TestPaper1;TestBook2 diff --git a/test/PoseidonGoldenTests/GoldenTestData/rectify/Wang_2020/Wang_2020.ssf b/test/PoseidonGoldenTests/GoldenTestData/rectify/Wang_2020/Wang_2020.ssf new file mode 100644 index 000000000..06b2766cb --- /dev/null +++ b/test/PoseidonGoldenTests/GoldenTestData/rectify/Wang_2020/Wang_2020.ssf @@ -0,0 +1,2 @@ +poseidon_IDs run_accession other_info_1 other_info_2 +SAMPLE1 ERR3518154 A B diff --git a/test/PoseidonGoldenTests/GoldenTestData/rectify/Wang_2020/sources.bib b/test/PoseidonGoldenTests/GoldenTestData/rectify/Wang_2020/sources.bib new file mode 100644 index 000000000..93693fbf3 --- /dev/null +++ b/test/PoseidonGoldenTests/GoldenTestData/rectify/Wang_2020/sources.bib @@ -0,0 +1,11 @@ +@article{Wang2020, + title = Test +} + +@article{TestPaper1, + title = TestPaper +} + +@book{TestBook2, + title = TestBook +} diff --git a/test/PoseidonGoldenTests/GoldenTestData/timetravel/Wang_2020-0.1.0/POSEIDON.yml b/test/PoseidonGoldenTests/GoldenTestData/timetravel/Wang_2020-0.1.0/POSEIDON.yml index 1e137405f..e7ff75a1a 100644 --- a/test/PoseidonGoldenTests/GoldenTestData/timetravel/Wang_2020-0.1.0/POSEIDON.yml +++ b/test/PoseidonGoldenTests/GoldenTestData/timetravel/Wang_2020-0.1.0/POSEIDON.yml @@ -2,16 +2,22 @@ poseidonVersion: 2.7.1 title: Wang_2020 description: Genetic data published in Wang et al. 2020, Plink test contributor: - - name: Ke Wang - email: wang@institute.org +- name: Ke Wang + email: wang@institute.org packageVersion: 0.1.0 lastModified: 2020-05-20 -bibFile: sources.bib genotypeData: format: PLINK genoFile: Wang_2020.bed + genoFileChkSum: ae66d851301f4a761b819f97ec28fa55 snpFile: Wang_2020.bim + snpFileChkSum: f96ea12084830fb43b3a24fd867f0ba7 indFile: Wang_2020.fam + indFileChkSum: 575ea54b293e297b5f354c910fe1610d snpSet: Other jannoFile: Wang_2020.janno -sequencingSourceFile: Wang_2020.ssf \ No newline at end of file +jannoFileChkSum: d271413712cf63767fc12dc3761de2e4 +sequencingSourceFile: Wang_2020.ssf +sequencingSourceFileChkSum: 76f24988a5440fea474308f9151548a3 +bibFile: sources.bib +bibFileChkSum: 67a9340931939f6d49fefe132c6ae984 diff --git a/test/PoseidonGoldenTests/GoldenTestsRunCommands.hs b/test/PoseidonGoldenTests/GoldenTestsRunCommands.hs index 4fb9b3f39..2d2b09c68 100644 --- a/test/PoseidonGoldenTests/GoldenTestsRunCommands.hs +++ b/test/PoseidonGoldenTests/GoldenTestsRunCommands.hs @@ -20,9 +20,11 @@ import Poseidon.CLI.Trident.List (ListEntity (..), ListOptions (..), RepoLocationSpec (..), runList) -import Poseidon.CLI.Trident.Rectify (ChecksumsToRectify (..), +import Poseidon.CLI.Trident.Modify (ChecksumsToModify (..), + ModifyOptions (..), PackageVersionUpdate (..), - RectifyOptions (..), + runModify) +import Poseidon.CLI.Trident.Rectify (RectifyOptions (..), runRectify) import Poseidon.CLI.Trident.Serve (ArchiveConfig (..), ArchiveSpec (..), @@ -46,6 +48,8 @@ import Poseidon.Core.GenotypeData (GenoDataSource (..), GenotypeFileSpec (..), GenotypeOutFormatSpec (..), SNPSetSpec (..)) +import Poseidon.Core.Package (PoseidonYamlStruct (..), + readPoseidonYaml) import Poseidon.Core.PoseidonVersion (VersionedFile (..), latestPoseidonVersion) import Poseidon.Core.ServerClient (AddColSpec (..), @@ -63,6 +67,7 @@ import Control.Exception (finally) import Control.Monad (forM_, unless, when) import Data.Either (fromRight) import Data.Function ((&)) +import Data.Maybe (isNothing) import qualified Data.Text as T import qualified Data.Text.IO as T import Data.Version (makeVersion) @@ -239,6 +244,8 @@ runCLICommands interactive testDir checkFilePath = do testPipelineGenoconvert testDir checkFilePath hPutStrLn stderr "--- rectify" testPipelineRectify testDir checkFilePath + hPutStrLn stderr "--- modify" + testPipelineModify testDir checkFilePath hPutStrLn stderr "--- forge" testPipelineForge testDir checkFilePath hPutStrLn stderr "--- chronicle & timetravel" @@ -647,66 +654,117 @@ testPipelineGenoconvert testDir checkFilePath = do testPipelineRectify :: FilePath -> FilePath -> IO () testPipelineRectify testDir checkFilePath = do - let rectifyOpts1 = RectifyOptions { - _rectifyBaseDirs = [testDir "init" "Schiffels"] - , _rectifyPoseidonVersion = Nothing - , _rectifyIgnorePoseidonVersion = False - , _rectifyPackageVersionUpdate = Just (PackageVersionUpdate Major (Just "test1")) - , _rectifyChecksums = ChecksumNone - , _rectifyNewContributors = Nothing - , _rectifyJannoRemoveEmptyCols = False - , _rectifyOnlyLatest = False + let rectifyDir = testDir "rectify" + noChecksumPac = rectifyDir "Schmid_2028" + changedPac = rectifyDir "Schiffels_2016" + unchangedPac = rectifyDir "Wang_2020" + noChecksumYaml = noChecksumPac "POSEIDON.yml" + changedYaml = changedPac "POSEIDON.yml" + unchangedYaml = unchangedPac "POSEIDON.yml" + copyDirectoryRecursive (testPacsDir "Schmid_2028") noChecksumPac + copyDirectoryRecursive (testPacsDir "Schiffels_2016") changedPac + copyDirectoryRecursive (testPacsDir "Wang_2020") unchangedPac + -- keep the bibliography syntactically valid while invalidating its checksum + -- this should make Schiffels_2016 get selected for rectification + appendFile (changedPac "sources.bib") "\n" + unchangedYamlBefore <- getChecksum unchangedYaml + let rectifyOpts = RectifyOptions { + _rectifyBaseDirs = [rectifyDir] + , _rectifyPackageVersionUpdate = Just (PackageVersionUpdate Major (Just "rectify test")) + , _rectifyNewContributors = Just [ContributorSpec "rectify" "rectify@example.org" Nothing] } - let action1 = testLog (runRectify rectifyOpts1) >> patchLastModified testDir ("init" "Schiffels" "POSEIDON.yml") - runAndChecksumFiles checkFilePath testDir action1 "rectify" [ + action = do + -- first run: rectify Schmid_2028 and Schiffels_2016, leave Wang_2020 untouched + noChecksumYamlStruct <- readPoseidonYaml noChecksumYaml + unless (isNothing $ _posYamlJannoFileChkSum noChecksumYamlStruct) $ + fail "Schmid_2028 seems to have checksums now: that renders this test invalid" + testLog $ runRectify rectifyOpts + patchLastModified testDir ("rectify" "Schmid_2028" "POSEIDON.yml") + patchLastModified testDir ("rectify" "Schiffels_2016" "POSEIDON.yml") + unchangedYamlAfter <- getChecksum unchangedYaml + unless (unchangedYamlBefore == unchangedYamlAfter) $ + fail "rectify modified a package with valid checksums" + -- second run: leave all untouched + changedYamlAfterFirstRun <- getChecksum changedYaml + testLog $ runRectify rectifyOpts + changedYamlAfterSecondRun <- getChecksum changedYaml + unless (changedYamlAfterFirstRun == changedYamlAfterSecondRun) $ + fail "rectify was not idempotent after repairing checksums" + runAndChecksumFiles checkFilePath testDir action "rectify" [ + "rectify" "Schmid_2028" "POSEIDON.yml" + , "rectify" "Schmid_2028" "CHANGELOG.md" + , "rectify" "Schiffels_2016" "POSEIDON.yml" + , "rectify" "Schiffels_2016" "CHANGELOG.md" + , "rectify" "Schiffels_2016" "sources.bib" + , "rectify" "Wang_2020" "POSEIDON.yml" + ] + +testPipelineModify :: FilePath -> FilePath -> IO () +testPipelineModify testDir checkFilePath = do + let modifyOpts1 = ModifyOptions { + _modifyBaseDirs = [testDir "init" "Schiffels"] + , _modifyPoseidonVersion = Nothing + , _modifyIgnorePoseidonVersion = False + , _modifyPackageVersionUpdate = Just (PackageVersionUpdate Major (Just "test1")) + , _modifyChecksums = ChecksumNone + , _modifyNewContributors = Nothing + , _modifyJannoRemoveEmptyCols = False + , _modifyOnlyLatest = False + , _modifyForce = False + } + let action1 = testLog (runModify modifyOpts1) >> patchLastModified testDir ("init" "Schiffels" "POSEIDON.yml") + runAndChecksumFiles checkFilePath testDir action1 "modify" [ "init" "Schiffels" "POSEIDON.yml" , "init" "Schiffels" "CHANGELOG.md" ] - let rectifyOpts2 = RectifyOptions { - _rectifyBaseDirs = [testDir "init" "Schiffels"] - , _rectifyPoseidonVersion = Just $ makeVersion [2,7,1] - , _rectifyIgnorePoseidonVersion = False - , _rectifyPackageVersionUpdate = Just (PackageVersionUpdate Minor (Just "test2")) - , _rectifyChecksums = ChecksumAll - , _rectifyNewContributors = Nothing - , _rectifyJannoRemoveEmptyCols = False - , _rectifyOnlyLatest = False + let modifyOpts2 = ModifyOptions { + _modifyBaseDirs = [testDir "init" "Schiffels"] + , _modifyPoseidonVersion = Just $ makeVersion [2,7,1] + , _modifyIgnorePoseidonVersion = False + , _modifyPackageVersionUpdate = Just (PackageVersionUpdate Minor (Just "test2")) + , _modifyChecksums = ChecksumAll + , _modifyNewContributors = Nothing + , _modifyJannoRemoveEmptyCols = False + , _modifyOnlyLatest = False + , _modifyForce = False } - let action2 = testLog (runRectify rectifyOpts2) >> patchLastModified testDir ("init" "Schiffels" "POSEIDON.yml") - runAndChecksumFiles checkFilePath testDir action2 "rectify" [ + let action2 = testLog (runModify modifyOpts2) >> patchLastModified testDir ("init" "Schiffels" "POSEIDON.yml") + runAndChecksumFiles checkFilePath testDir action2 "modify" [ "init" "Schiffels" "POSEIDON.yml" , "init" "Schiffels" "CHANGELOG.md" ] - let rectifyOpts3 = RectifyOptions { - _rectifyBaseDirs = [testDir "init" "Schiffels"] - , _rectifyPoseidonVersion = Nothing - , _rectifyIgnorePoseidonVersion = False - , _rectifyPackageVersionUpdate = Just (PackageVersionUpdate Patch Nothing) - , _rectifyChecksums = ChecksumNone - , _rectifyNewContributors = Just [ + let modifyOpts3 = ModifyOptions { + _modifyBaseDirs = [testDir "init" "Schiffels"] + , _modifyPoseidonVersion = Nothing + , _modifyIgnorePoseidonVersion = False + , _modifyPackageVersionUpdate = Just (PackageVersionUpdate Patch Nothing) + , _modifyChecksums = ChecksumNone + , _modifyNewContributors = Just [ ContributorSpec "Josiah Carberry" "carberry@brown.edu" (Just $ ORCID {_orcidNums = "000000021825009", _orcidChecksum = '7'}) , ContributorSpec "Herbert Testmann" "herbert@testmann.tw" Nothing ] - , _rectifyJannoRemoveEmptyCols = False - , _rectifyOnlyLatest = False + , _modifyJannoRemoveEmptyCols = False + , _modifyOnlyLatest = False + , _modifyForce = False } - let action3 = testLog (runRectify rectifyOpts3) >> patchLastModified testDir ("init" "Schiffels" "POSEIDON.yml") - runAndChecksumFiles checkFilePath testDir action3 "rectify" [ + let action3 = testLog (runModify modifyOpts3) >> patchLastModified testDir ("init" "Schiffels" "POSEIDON.yml") + runAndChecksumFiles checkFilePath testDir action3 "modify" [ "init" "Schiffels" "POSEIDON.yml" , "init" "Schiffels" "CHANGELOG.md" ] - let rectifyOpts4 = RectifyOptions { - _rectifyBaseDirs = [testDir "init" "Schiffels"] - , _rectifyPoseidonVersion = Nothing - , _rectifyIgnorePoseidonVersion = False - , _rectifyPackageVersionUpdate = Nothing - , _rectifyChecksums = ChecksumAll - , _rectifyNewContributors = Nothing - , _rectifyJannoRemoveEmptyCols = True - , _rectifyOnlyLatest = False + let modifyOpts4 = ModifyOptions { + _modifyBaseDirs = [testDir "init" "Schiffels"] + , _modifyPoseidonVersion = Nothing + , _modifyIgnorePoseidonVersion = False + , _modifyPackageVersionUpdate = Nothing + , _modifyChecksums = ChecksumAll + , _modifyNewContributors = Nothing + , _modifyJannoRemoveEmptyCols = True + , _modifyOnlyLatest = False + , _modifyForce = False } - let action4 = testLog (runRectify rectifyOpts4) >> patchLastModified testDir ("init" "Schiffels" "POSEIDON.yml") - runAndChecksumFiles checkFilePath testDir action4 "rectify" [ + let action4 = testLog (runModify modifyOpts4) >> patchLastModified testDir ("init" "Schiffels" "POSEIDON.yml") + runAndChecksumFiles checkFilePath testDir action4 "modify" [ "init" "Schiffels" "POSEIDON.yml" , "init" "Schiffels" "CHANGELOG.md" , "init" "Schiffels" "Schiffels.janno" diff --git a/test/testDat/testPackages/ancient/Wang_2020/POSEIDON.yml b/test/testDat/testPackages/ancient/Wang_2020/POSEIDON.yml index 1e137405f..e7ff75a1a 100644 --- a/test/testDat/testPackages/ancient/Wang_2020/POSEIDON.yml +++ b/test/testDat/testPackages/ancient/Wang_2020/POSEIDON.yml @@ -2,16 +2,22 @@ poseidonVersion: 2.7.1 title: Wang_2020 description: Genetic data published in Wang et al. 2020, Plink test contributor: - - name: Ke Wang - email: wang@institute.org +- name: Ke Wang + email: wang@institute.org packageVersion: 0.1.0 lastModified: 2020-05-20 -bibFile: sources.bib genotypeData: format: PLINK genoFile: Wang_2020.bed + genoFileChkSum: ae66d851301f4a761b819f97ec28fa55 snpFile: Wang_2020.bim + snpFileChkSum: f96ea12084830fb43b3a24fd867f0ba7 indFile: Wang_2020.fam + indFileChkSum: 575ea54b293e297b5f354c910fe1610d snpSet: Other jannoFile: Wang_2020.janno -sequencingSourceFile: Wang_2020.ssf \ No newline at end of file +jannoFileChkSum: d271413712cf63767fc12dc3761de2e4 +sequencingSourceFile: Wang_2020.ssf +sequencingSourceFileChkSum: 76f24988a5440fea474308f9151548a3 +bibFile: sources.bib +bibFileChkSum: 67a9340931939f6d49fefe132c6ae984