diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index b0f9460..50df777 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -1,73 +1,73 @@ -name: Publish Python Package - -on: - push: - branches: [main] - tags: - - '[0-9]+.[0-9]+.[0-9]+' - - '[0-9]+.[0-9]+.[0-9]+-alpha.[0-9]+' - - '[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+' - - '[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+' - pull_request: - branches: [main] - -jobs: - release-build: - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Install UV - run: | - curl -LsSf https://astral.sh/uv/install.sh | sh - echo "$HOME/.local/bin" >> $GITHUB_PATH - - - name: Build Package with UV - run: | - uv build - - - name: Upload Built Package - uses: actions/upload-artifact@v4 - with: - name: release-dists - path: dist/ - - pypi-publish: - if: startsWith(github.ref, 'refs/tags/') - runs-on: ubuntu-latest - needs: [release-build] - permissions: - id-token: write # OIDC auth for PyPI - contents: read - steps: - - name: Retrieve Built Package - uses: actions/download-artifact@v4 - with: - name: release-dists - path: dist/ - - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.12.4 - - github-release: - if: startsWith(github.ref, 'refs/tags/') - runs-on: ubuntu-latest - needs: [release-build] - permissions: - contents: write - steps: - - name: Retrieve Built Package - uses: actions/download-artifact@v4 - with: - name: release-dists - path: dist/ - - - name: Create GitHub Release - uses: softprops/action-gh-release@v2 - with: - files: dist/* - tag_name: ${{ github.ref_name }} - name: "Release ${{ github.ref_name }}" - draft: false - prerelease: ${{ contains(github.ref_name, '-beta') || contains(github.ref_name, '-alpha') || contains(github.ref_name, '-rc') }} +name: Publish Python Package + +on: + push: + branches: [main] + tags: + - '[0-9]+.[0-9]+.[0-9]+' + - '[0-9]+.[0-9]+.[0-9]+-alpha.[0-9]+' + - '[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+' + - '[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+' + pull_request: + branches: [main] + +jobs: + release-build: + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Install UV + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Build Package with UV + run: | + uv build + + - name: Upload Built Package + uses: actions/upload-artifact@v4 + with: + name: release-dists + path: dist/ + + pypi-publish: + if: startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + needs: [release-build] + permissions: + id-token: write # OIDC auth for PyPI + contents: read + steps: + - name: Retrieve Built Package + uses: actions/download-artifact@v4 + with: + name: release-dists + path: dist/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@v1.12.4 + + github-release: + if: startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + needs: [release-build] + permissions: + contents: write + steps: + - name: Retrieve Built Package + uses: actions/download-artifact@v4 + with: + name: release-dists + path: dist/ + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + files: dist/* + tag_name: ${{ github.ref_name }} + name: "Release ${{ github.ref_name }}" + draft: false + prerelease: ${{ contains(github.ref_name, '-beta') || contains(github.ref_name, '-alpha') || contains(github.ref_name, '-rc') }} diff --git a/.gitignore b/.gitignore index 0d7571e..649f1e0 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,10 @@ src/*.egg-info/ **/artifacts/ *.ini -*.env \ No newline at end of file +*.env + +# User/Agent Documentation +JOBS_DOCUMENTATION.md +AGENTS.md +.antigravity/ +reports/ \ No newline at end of file diff --git a/README.md b/README.md index e854c5a..5c5a09f 100644 --- a/README.md +++ b/README.md @@ -14,126 +14,90 @@ It uses `.dmtri_jobs.json` per host to track async job state and integrates with --- -## Quick Start +## Installation & Setup -You can use `dmtri` in two ways, depending on your preference: - -### Option 1: Using `uvx` - -You can run commands using: +### 1. Install or Run +You can run `dmtri` directly or install it as a tool: +#### Run without installing (via `uvx`) ```bash -uvx dmtri [options] - +uvx dmtri ``` ---- - -### Option 2: Global (Tool-based) — Ideal for global CLI use - +#### Install as a global tool ```bash +# Using uv (recommended) uv tool install dmtri -``` - -This makes `dmtri` available globally: - -```bash -dmtri [options] +# Using pip +pip install dmtri ``` -You can update it any time using: +### 2. Configure +Once installed, set up your local configuration directory: ```bash -uv tool upgrade dmtri +dmtri init ``` ---- - -### 2. Configure inventory - -Edit the file: `src/inventory/hosts.ini` - -Example: - -```ini -[seedpsd_nodes] - ansible_user= ansible_ssh_pass="" ansible_connection=ssh ansible_become=false ansible_ssh_common_args='-o StrictHostKeyChecking=no' +This creates a config folder (e.g., `~/.config/dmtri/` on Linux) with a template `hosts.ini` and all standard playbooks. -[seedpsd_nodes:vars] -seedpsd_dir=/home//seedpsd +#### Edit your Inventory +Use `nano` (or any text editor) to fill in your EIDA server details: -[wf_catalogue] - ansible_user= ansible_ssh_pass="" ansible_connection=ssh ansible_become=false ansible_ssh_common_args='-o StrictHostKeyChecking=no' - -[wf_catalogue:vars] -collector_dir=/home//Programs/wfcatalogue/wfcatalog/collector2 - -[ws_availability] - ansible_user= ansible_ssh_pass="" ansible_connection=ssh ansible_become=false ansible_ssh_common_args='-o StrictHostKeyChecking=no' +```bash +nano ~/.config/dmtri/hosts.ini +``` -[ws_availability:vars] -availability_dir=/home//Programs/ws-availability/views -mongo_user= -mongo_pass= -mongo_authdb= +#### Verify Connectivity +Check if your servers are reachable: +```bash +dmtri doctor ``` --- -### 3. Run example jobs +## Usage -By default, this command runs **all refresh playbooks** (seedpsd, wfcatalog, and availability): +### Run Refresh Jobs +Trigger refresh playbooks across your nodes: ```bash -uv run dmtri refresh --network HL --station ATH --starttime 2024-01-01T00:00:00 --endtime 2024-01-02T00:00:00 - +dmtri refresh --network HL --station ATH --starttime 2024-01-01 ``` ---- - -### 4. Track jobs across hosts +### Track Job Status +Monitor the status of asynchronous jobs: ```bash -uv run dmtri track -``` - -Shows per-host job status: -``` -Job ID: j123... Type: seedpsd Station: ATH Status: ✅ Finished -Job ID: manual-... Type: availability Status: ✅ Manual (not tracked) +dmtri track ``` --- -### 5. Troubleshooting with `dmtri doctor` +## Customization -Check SSH connectivity to your configured hosts using: +### Local Playbooks +`dmtri init` copies all playbooks to your config directory. You can edit them to change automation logic: ```bash -uv run dmtri doctor +# Location: ~/.config/dmtri/playbooks/ +nano ~/.config/dmtri/playbooks/refresh/availability_refresh.yml ``` -This uses the Ansible `ping` module to test access to all servers in your inventory. - -#### Options +`dmtri` prioritizes these local playbooks over the bundled defaults. -- `--verbose` — Show full output from each host - -#### Example +### Inventory Overrides +Override the inventory at runtime: ```bash -uv run dmtri doctor --verbose +dmtri doctor --inventory ./my_custom_hosts.ini ``` -If no response is received, ensure: - -- You're connected to the VPN (if applicable) -- SSH access is configured correctly in your inventory file -- Host IPs and credentials are reachable and valid +--- -### Operation Details +## Operation Details `dmtri` supports two main operations for both metadata and data: `refresh` and `clean`. These affect the following components: diff --git a/badges/coverage.svg b/badges/coverage.svg index ee07d4c..0bc8393 100644 --- a/badges/coverage.svg +++ b/badges/coverage.svg @@ -1,21 +1,21 @@ - - - - - - - - - - - - - - - - coverage - coverage - 96% - 96% - - + + + + + + + + + + + + + + + + coverage + coverage + 96% + 96% + + diff --git a/pyproject.toml b/pyproject.toml index aebd643..c845cb5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,8 +14,8 @@ dependencies = [ ] readme = "README.md" requires-python = ">=3.8" -[tool.uv] -dev-dependencies = [ +[dependency-groups] +dev = [ "pytest", "pytest-cov" ] @@ -23,6 +23,10 @@ dev-dependencies = [ [project.scripts] dmtri = "dmtri.cli:main" + +[tool.setuptools.package-data] +dmtri = ["data/**/*"] + [build-system] requires = ["setuptools>=62.0", "wheel"] build-backend = "setuptools.build_meta" diff --git a/src/dmtri/cli.py b/src/dmtri/cli.py index 42a3b06..d4886a7 100644 --- a/src/dmtri/cli.py +++ b/src/dmtri/cli.py @@ -5,9 +5,9 @@ import logging from dmtri.execution_plan import print_execution_plan -from dmtri.utils import get_version, validate_and_normalize_args +from dmtri.utils import get_version, validate_and_normalize_args , parse_flexible_date from dmtri.hooks.hooks import run_hook -from dmtri.paths import COMMAND_PLAYBOOKS +from dmtri.paths import COMMAND_PLAYBOOKS, ANSIBLE_QUIET_CFG, ANSIBLE_TRACK_CFG, ANSIBLE_VERBOSE_CFG, get_inventory_path logging.basicConfig(level=logging.INFO, format="%(message)s") logger = logging.getLogger(__name__) @@ -26,6 +26,7 @@ def main(): action="version", version=f"%(prog)s version {get_version()} — See {PROJECT_URL} for updates or to report issues." ) + parser.add_argument("-i", "--inventory", help="Path to custom Ansible inventory file") subparsers = parser.add_subparsers(dest="command") @@ -34,10 +35,18 @@ def main(): shared.add_argument("-s", "--station", nargs="+", default=["*"], help="FDSN station code(s), supports wildcards") shared.add_argument("-l", "--location", nargs="+", default=["*"], help="FDSN location code(s), supports wildcards") shared.add_argument("-c", "--channel", nargs="+", default=["*"], help="FDSN channel code(s), supports wildcards") - shared.add_argument("-S", "--starttime", required=True, help="Start time (ISO 8601 format)") - - now_str = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S") - shared.add_argument("-E", "--endtime", default=now_str, help="End time (default: now in UTC)") + shared.add_argument( + "-S", "--starttime", "--start", + type=parse_flexible_date, + required=True, + help="Start time (e.g. '2025-07-01', '2025-07-01T12:00:00', 'yesterday', '-2d')", + ) + shared.add_argument( + "-E", "--endtime", "--end", + type=parse_flexible_date, + default="now", + help="End time (default: now in UTC)", + ) shared.add_argument("--type", default="data,metadata", help="Comma-separated types to process (data, metadata)") shared.add_argument("--no-confirm", action="store_true", help="Skip confirmation prompt before executing") shared.add_argument("--debug", action="store_true", help="Show verbose output from Ansible") @@ -48,18 +57,25 @@ def main(): doctor_parser = subparsers.add_parser("doctor", help="Check SSH connectivity to all inventory hosts") doctor_parser.add_argument("-v", "--verbose", action="store_true", help="Show full Ansible output per host") + init_parser = subparsers.add_parser("init", help="Scaffold user config directory with bundled playbooks and inventory") + init_parser.add_argument("--force", action="store_true", help="Overwrite existing user config files") + args = parser.parse_args() if args.command == "track": - cfg_file = "ansible_verbose.cfg" + cfg_path = ANSIBLE_TRACK_CFG else: - cfg_file = "ansible_verbose.cfg" if getattr(args, "debug", False) else "ansible_quiet.cfg" + cfg_path = ANSIBLE_VERBOSE_CFG if getattr(args, "debug", False) else ANSIBLE_QUIET_CFG + + os.environ["ANSIBLE_CONFIG"] = str(cfg_path.resolve()) + inventory_path = get_inventory_path(args.inventory) - cfg_path = os.path.abspath(cfg_file) - os.environ["ANSIBLE_CONFIG"] = cfg_path + if args.command == "init": + from dmtri.init import run_init + return run_init(force=args.force) if args.command == "doctor": from dmtri.doctor import run_diagnostics - return run_diagnostics(verbose=args.verbose) + return run_diagnostics(verbose=args.verbose, inventory=inventory_path) elif args.command in ("refresh", "clean"): start_dt, end_dt, types = validate_and_normalize_args(args, parser) @@ -69,12 +85,13 @@ def main(): "station": args.station, "location": args.location, "channel": args.channel, - "starttime": args.starttime, - "endtime": args.endtime, + "starttime": args.starttime.isoformat(), + "endtime": args.endtime.isoformat(), "type": [t.strip() for t in args.type.split(",")], "debug": args.debug, } + if args.command == "refresh": selected_types = vars_to_pass["type"] valid_types = set(COMMAND_PLAYBOOKS["refresh"].keys()) @@ -103,7 +120,7 @@ def main(): sys.exit(0) for pb in playbooks: - run_hook(pb, vars_to_pass) + run_hook(pb, vars_to_pass, inventory=inventory_path) elif args.command == "track": playbooks = COMMAND_PLAYBOOKS.get("track", []) @@ -111,7 +128,7 @@ def main(): print("No playbooks configured for command 'track'.") sys.exit(1) for pb in playbooks: - run_hook(pb, {}) + run_hook(pb, {}, inventory=inventory_path) else: parser.print_help() diff --git a/ansible_quiet.cfg b/src/dmtri/data/ansible_quiet.cfg similarity index 96% rename from ansible_quiet.cfg rename to src/dmtri/data/ansible_quiet.cfg index 8d91fdc..9bd5020 100644 --- a/ansible_quiet.cfg +++ b/src/dmtri/data/ansible_quiet.cfg @@ -32,10 +32,10 @@ nocolor=False ;become_password_file= # (pathspec) Colon-separated paths in which Ansible will search for Become Plugins. -;become_plugins=/home/nsokos/.ansible/plugins/become:/usr/share/ansible/plugins/become +;become_plugins=/home/nikos/.ansible/plugins/become:/usr/share/ansible/plugins/become # (string) Chooses which cache plugin to use, the default 'memory' is ephemeral. -;fact_caching=memory +fact_caching=memory # (string) Defines connection or path information for the cache plugin. ;fact_caching_connection= @@ -54,7 +54,7 @@ nocolor=False # (pathspec) Colon-separated paths in which Ansible will search for collections content. Collections must be in nested *subdirectories*, not directly in these directories. For example, if ``COLLECTIONS_PATHS`` includes ``'{{ ANSIBLE_HOME ~ "/collections" }}'``, and you want to add ``my.collection`` to that directory, it must be saved as ``'{{ ANSIBLE_HOME} ~ "/collections/ansible_collections/my/collection" }}'``. -;collections_path=/home/nsokos/.ansible/collections:/usr/share/ansible/collections +;collections_path=/home/nikos/.ansible/collections:/usr/share/ansible/collections # (boolean) A boolean to enable or disable scanning the sys.path for installed collections. ;collections_scan_sys_path=True @@ -63,7 +63,7 @@ nocolor=False ;connection_password_file= # (pathspec) Colon-separated paths in which Ansible will search for Action Plugins. -;action_plugins=/home/nsokos/.ansible/plugins/action:/usr/share/ansible/plugins/action +;action_plugins=/home/nikos/.ansible/plugins/action:/usr/share/ansible/plugins/action # (boolean) When enabled, this option allows lookup plugins (whether used in variables as ``{{lookup('foo')}}`` or as a loop as with_foo) to return data that is not marked 'unsafe'. # By default, such data is marked as unsafe to prevent the templating engine from evaluating any jinja2 templating language, as this could represent a security risk. This option is provided to allow for backward compatibility, however, users should first consider adding allow_unsafe=True to any lookups that may be expected to contain data that may be run through the templating engine late. @@ -76,16 +76,16 @@ nocolor=False ;ask_vault_pass=False # (pathspec) Colon-separated paths in which Ansible will search for Cache Plugins. -;cache_plugins=/home/nsokos/.ansible/plugins/cache:/usr/share/ansible/plugins/cache +;cache_plugins=/home/nikos/.ansible/plugins/cache:/usr/share/ansible/plugins/cache # (pathspec) Colon-separated paths in which Ansible will search for Callback Plugins. -;callback_plugins=/home/nsokos/.ansible/plugins/callback:/usr/share/ansible/plugins/callback +;callback_plugins=/home/nikos/.ansible/plugins/callback:/usr/share/ansible/plugins/callback # (pathspec) Colon-separated paths in which Ansible will search for Cliconf Plugins. -;cliconf_plugins=/home/nsokos/.ansible/plugins/cliconf:/usr/share/ansible/plugins/cliconf +;cliconf_plugins=/home/nikos/.ansible/plugins/cliconf:/usr/share/ansible/plugins/cliconf # (pathspec) Colon-separated paths in which Ansible will search for Connection Plugins. -;connection_plugins=/home/nsokos/.ansible/plugins/connection:/usr/share/ansible/plugins/connection +;connection_plugins=/home/nikos/.ansible/plugins/connection:/usr/share/ansible/plugins/connection # (boolean) Toggles debug output in Ansible. This is *very* verbose and can hinder multiprocessing. Debug output can also include secret information despite no_log settings being enabled, which means debug mode should not be used in production. ;debug=False @@ -94,7 +94,7 @@ nocolor=False ;executable=/bin/sh # (pathspec) Colon-separated paths in which Ansible will search for Jinja2 Filter Plugins. -;filter_plugins=/home/nsokos/.ansible/plugins/filter:/usr/share/ansible/plugins/filter +;filter_plugins=/home/nikos/.ansible/plugins/filter:/usr/share/ansible/plugins/filter # (boolean) This option controls if notified handlers run on a host even if a failure occurs on that host. # When false, the handlers will not run if a failure has occurred on a host. @@ -123,14 +123,14 @@ nocolor=False ;inventory=/etc/ansible/hosts # (pathspec) Colon-separated paths in which Ansible will search for HttpApi Plugins. -;httpapi_plugins=/home/nsokos/.ansible/plugins/httpapi:/usr/share/ansible/plugins/httpapi +;httpapi_plugins=/home/nikos/.ansible/plugins/httpapi:/usr/share/ansible/plugins/httpapi # (float) This sets the interval (in seconds) of Ansible internal processes polling each other. Lower values improve performance with large playbooks at the expense of extra CPU load. Higher values are more suitable for Ansible usage in automation scenarios when UI responsiveness is not required but CPU usage might be a concern. # The default corresponds to the value hardcoded in Ansible <= 2.1 ;internal_poll_interval=0.001 # (pathspec) Colon-separated paths in which Ansible will search for Inventory Plugins. -;inventory_plugins=/home/nsokos/.ansible/plugins/inventory:/usr/share/ansible/plugins/inventory +;inventory_plugins=/home/nikos/.ansible/plugins/inventory:/usr/share/ansible/plugins/inventory # (string) This is a developer-specific feature that allows enabling additional Jinja2 extensions. # See the Jinja2 documentation for details. If you do not know what these do, you probably don't need to change this setting :) @@ -147,7 +147,7 @@ nocolor=False ;bin_ansible_callbacks=False # (tmppath) Temporary directory for Ansible to use on the controller. -;local_tmp=/home/nsokos/.ansible/tmp +local_tmp=/home/nikos/.ansible/tmp # (list) List of logger names to filter out of the log file. ;log_filter= @@ -157,7 +157,7 @@ nocolor=False ;log_path= # (pathspec) Colon-separated paths in which Ansible will search for Lookup Plugins. -;lookup_plugins=/home/nsokos/.ansible/plugins/lookup:/usr/share/ansible/plugins/lookup +;lookup_plugins=/home/nikos/.ansible/plugins/lookup:/usr/share/ansible/plugins/lookup # (string) Sets the macro for the 'ansible_managed' variable available for :ref:`ansible_collections.ansible.builtin.template_module` and :ref:`ansible_collections.ansible.windows.win_template_module`. This is only relevant to those two modules. ;ansible_managed=Ansible managed @@ -172,13 +172,13 @@ nocolor=False ;module_name=command # (pathspec) Colon-separated paths in which Ansible will search for Modules. -;library=/home/nsokos/.ansible/plugins/modules:/usr/share/ansible/plugins/modules +;library=/home/nikos/.ansible/plugins/modules:/usr/share/ansible/plugins/modules # (pathspec) Colon-separated paths in which Ansible will search for Module utils files, which are shared by modules. -;module_utils=/home/nsokos/.ansible/plugins/module_utils:/usr/share/ansible/plugins/module_utils +;module_utils=/home/nikos/.ansible/plugins/module_utils:/usr/share/ansible/plugins/module_utils # (pathspec) Colon-separated paths in which Ansible will search for Netconf Plugins. -;netconf_plugins=/home/nsokos/.ansible/plugins/netconf:/usr/share/ansible/plugins/netconf +;netconf_plugins=/home/nikos/.ansible/plugins/netconf:/usr/share/ansible/plugins/netconf # (boolean) Toggle Ansible's display and logging of task details, mainly used to avoid security disclosures. ;no_log=False @@ -209,18 +209,18 @@ nocolor=False ;remote_user= # (pathspec) Colon-separated paths in which Ansible will search for Roles. -;roles_path=/home/nsokos/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles +;roles_path=/home/nikos/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles # (string) Set the main callback used to display Ansible output. You can only have one at a time. # You can have many other callbacks, but just one can be in charge of stdout. # See :ref:`callback_plugins` for a list of available options. -stdout_callback=null +stdout_callback=default # (string) Set the default strategy used for plays. ;strategy=linear # (pathspec) Colon-separated paths in which Ansible will search for Strategy Plugins. -;strategy_plugins=/home/nsokos/.ansible/plugins/strategy:/usr/share/ansible/plugins/strategy +;strategy_plugins=/home/nikos/.ansible/plugins/strategy:/usr/share/ansible/plugins/strategy # (boolean) Toggle the use of "su" for tasks. ;su=False @@ -229,10 +229,10 @@ stdout_callback=null ;syslog_facility=LOG_USER # (pathspec) Colon-separated paths in which Ansible will search for Terminal Plugins. -;terminal_plugins=/home/nsokos/.ansible/plugins/terminal:/usr/share/ansible/plugins/terminal +;terminal_plugins=/home/nikos/.ansible/plugins/terminal:/usr/share/ansible/plugins/terminal # (pathspec) Colon-separated paths in which Ansible will search for Jinja2 Test Plugins. -;test_plugins=/home/nsokos/.ansible/plugins/test:/usr/share/ansible/plugins/test +;test_plugins=/home/nikos/.ansible/plugins/test:/usr/share/ansible/plugins/test # (integer) This is the default timeout for connection plugins to use. ;timeout=10 @@ -246,7 +246,7 @@ stdout_callback=null ;error_on_undefined_vars=True # (pathspec) Colon-separated paths in which Ansible will search for Vars Plugins. -;vars_plugins=/home/nsokos/.ansible/plugins/vars:/usr/share/ansible/plugins/vars +;vars_plugins=/home/nikos/.ansible/plugins/vars:/usr/share/ansible/plugins/vars # (string) The vault_id to use for encrypting by default. If multiple vault_ids are provided, this specifies which to use for encryption. The ``--encrypt-vault-id`` CLI option overrides the configured value. ;vault_encrypt_identity= @@ -279,13 +279,13 @@ deprecation_warnings=False ;display_args_to_stdout=False # (boolean) Toggle to control displaying skipped task/host entries in a task in the default callback. -display_skipped_hosts=False +display_skipped_hosts=True # (string) Root docsite URL used to generate docs URLs in warning/error text; must be an absolute URL with a valid scheme and trailing slash. ;docsite_root_url=https://docs.ansible.com/ansible-core/ # (pathspec) Colon-separated paths in which Ansible will search for Documentation Fragments Plugins. -;doc_fragment_plugins=/home/nsokos/.ansible/plugins/doc_fragments:/usr/share/ansible/plugins/doc_fragments +;doc_fragment_plugins=/home/nikos/.ansible/plugins/doc_fragments:/usr/share/ansible/plugins/doc_fragments # (string) By default, Ansible will issue a warning when a duplicate dict key is encountered in YAML. # These warnings can be silenced by adjusting this setting to False. @@ -435,7 +435,7 @@ retry_files_enabled=False display_failed_stderr=False # (bool) Toggle to control displaying 'ok' task/host results in a task. -display_ok_hosts=False +display_ok_hosts=True # (bool) Configure the result format to be more readable. # When O(result_format) is set to V(yaml) this option defaults to V(true), and defaults to V(false) when configured to V(json). @@ -519,7 +519,7 @@ display_ok_hosts=False ;connect_timeout=30 # (path) Path to the socket to be used by the connection persistence system. -;control_path_dir=/home/nsokos/.ansible/pc +;control_path_dir=/home/nikos/.ansible/pc [connection] @@ -617,7 +617,7 @@ display_ok_hosts=False # (path) The directory that stores cached responses from a Galaxy server. # This is only used by the ``ansible-galaxy collection install`` and ``download`` commands. # Cache files inside this dir will be ignored if they are world writable. -;cache_dir=/home/nsokos/.ansible/galaxy_cache +;cache_dir=/home/nikos/.ansible/galaxy_cache # (bool) whether ``ansible-galaxy collection install`` should warn about ``--collections-path`` missing from configured :ref:`collections_paths`. ;collections_path_warning=True @@ -671,7 +671,7 @@ display_ok_hosts=False ;server_timeout=60 # (path) Local path to galaxy access token file -;token_path=/home/nsokos/.ansible/galaxy_token +;token_path=/home/nikos/.ansible/galaxy_token [inventory] diff --git a/src/dmtri/data/ansible_track.cfg b/src/dmtri/data/ansible_track.cfg new file mode 100644 index 0000000..739c21f --- /dev/null +++ b/src/dmtri/data/ansible_track.cfg @@ -0,0 +1,712 @@ +[defaults] +# (boolean) By default, Ansible will issue a warning when received from a task action (module or action plugin). +# These warnings can be silenced by adjusting this setting to False. +;action_warnings=True + +# (list) Accept a list of cowsay templates that are 'safe' to use, set to an empty list if you want to enable all installed templates. +;cowsay_enabled_stencils=bud-frogs, bunny, cheese, daemon, default, dragon, elephant-in-snake, elephant, eyes, hellokitty, kitty, luke-koala, meow, milk, moofasa, moose, ren, sheep, small, stegosaurus, stimpy, supermilker, three-eyes, turkey, turtle, tux, udder, vader-koala, vader, www + +# (string) Specify a custom cowsay path or swap in your cowsay implementation of choice. +;cowpath= + +# (string) This allows you to choose a specific cowsay stencil for the banners or use 'random' to cycle through them. +;cow_selection=default + +# (boolean) This option forces color mode even when running without a TTY or the "nocolor" setting is True. +;force_color=False + +# (path) The default root path for Ansible config files on the controller. +;home=~/.ansible + +# (boolean) This setting allows suppressing colorizing output, which is used to give a better indication of failure and status information. +;nocolor=False + +# (boolean) If you have cowsay installed but want to avoid the 'cows' (why????), use this. +;nocows=False + +# (boolean) Sets the default value for the any_errors_fatal keyword, if True, Task failures will be considered fatal errors. +;any_errors_fatal=False + +# (path) The password file to use for the become plugin. ``--become-password-file``. +# If executable, it will be run and the resulting stdout will be used as the password. +;become_password_file= + +# (pathspec) Colon-separated paths in which Ansible will search for Become Plugins. +;become_plugins=/home/nikos/.ansible/plugins/become:/usr/share/ansible/plugins/become + +# (string) Chooses which cache plugin to use, the default 'memory' is ephemeral. +fact_caching=memory + +# (string) Defines connection or path information for the cache plugin. +;fact_caching_connection= + +# (string) Prefix to use for cache plugin files/tables. +;fact_caching_prefix=ansible_facts + +# (integer) Expiration timeout for the cache plugin data. +;fact_caching_timeout=86400 + +# (list) List of enabled callbacks, not all callbacks need enabling, but many of those shipped with Ansible do as we don't want them activated by default. +;callbacks_enabled= + +# (string) When a collection is loaded that does not support the running Ansible version (with the collection metadata key `requires_ansible`). +;collections_on_ansible_version_mismatch=warning + +# (pathspec) Colon-separated paths in which Ansible will search for collections content. Collections must be in nested *subdirectories*, not directly in these directories. For example, if ``COLLECTIONS_PATHS`` includes ``'{{ ANSIBLE_HOME ~ "/collections" }}'``, and you want to add ``my.collection`` to that directory, it must be saved as ``'{{ ANSIBLE_HOME} ~ "/collections/ansible_collections/my/collection" }}'``. + +;collections_path=/home/nikos/.ansible/collections:/usr/share/ansible/collections + +# (boolean) A boolean to enable or disable scanning the sys.path for installed collections. +;collections_scan_sys_path=True + +# (path) The password file to use for the connection plugin. ``--connection-password-file``. +;connection_password_file= + +# (pathspec) Colon-separated paths in which Ansible will search for Action Plugins. +;action_plugins=/home/nikos/.ansible/plugins/action:/usr/share/ansible/plugins/action + +# (boolean) When enabled, this option allows lookup plugins (whether used in variables as ``{{lookup('foo')}}`` or as a loop as with_foo) to return data that is not marked 'unsafe'. +# By default, such data is marked as unsafe to prevent the templating engine from evaluating any jinja2 templating language, as this could represent a security risk. This option is provided to allow for backward compatibility, however, users should first consider adding allow_unsafe=True to any lookups that may be expected to contain data that may be run through the templating engine late. +;allow_unsafe_lookups=False + +# (boolean) This controls whether an Ansible playbook should prompt for a login password. If using SSH keys for authentication, you probably do not need to change this setting. +;ask_pass=False + +# (boolean) This controls whether an Ansible playbook should prompt for a vault password. +;ask_vault_pass=False + +# (pathspec) Colon-separated paths in which Ansible will search for Cache Plugins. +;cache_plugins=/home/nikos/.ansible/plugins/cache:/usr/share/ansible/plugins/cache + +# (pathspec) Colon-separated paths in which Ansible will search for Callback Plugins. +;callback_plugins=/home/nikos/.ansible/plugins/callback:/usr/share/ansible/plugins/callback + +# (pathspec) Colon-separated paths in which Ansible will search for Cliconf Plugins. +;cliconf_plugins=/home/nikos/.ansible/plugins/cliconf:/usr/share/ansible/plugins/cliconf + +# (pathspec) Colon-separated paths in which Ansible will search for Connection Plugins. +;connection_plugins=/home/nikos/.ansible/plugins/connection:/usr/share/ansible/plugins/connection + +# (boolean) Toggles debug output in Ansible. This is *very* verbose and can hinder multiprocessing. Debug output can also include secret information despite no_log settings being enabled, which means debug mode should not be used in production. +;debug=False + +# (string) This indicates the command to use to spawn a shell under, which is required for Ansible's execution needs on a target. Users may need to change this in rare instances when shell usage is constrained, but in most cases, it may be left as is. +;executable=/bin/sh + +# (pathspec) Colon-separated paths in which Ansible will search for Jinja2 Filter Plugins. +;filter_plugins=/home/nikos/.ansible/plugins/filter:/usr/share/ansible/plugins/filter + +# (boolean) This option controls if notified handlers run on a host even if a failure occurs on that host. +# When false, the handlers will not run if a failure has occurred on a host. +# This can also be set per play or on the command line. See Handlers and Failure for more details. +;force_handlers=False + +# (integer) Maximum number of forks Ansible will use to execute tasks on target hosts. +;forks=5 + +# (string) This setting controls the default policy of fact gathering (facts discovered about remote systems). +# This option can be useful for those wishing to save fact gathering time. Both 'smart' and 'explicit' will use the cache plugin. +gathering=explicit + +# (string) This setting controls how duplicate definitions of dictionary variables (aka hash, map, associative array) are handled in Ansible. +# This does not affect variables whose values are scalars (integers, strings) or arrays. +# **WARNING**, changing this setting is not recommended as this is fragile and makes your content (plays, roles, collections) nonportable, leading to continual confusion and misuse. Don't change this setting unless you think you have an absolute need for it. +# We recommend avoiding reusing variable names and relying on the ``combine`` filter and ``vars`` and ``varnames`` lookups to create merged versions of the individual variables. In our experience, this is rarely needed and is a sign that too much complexity has been introduced into the data structures and plays. +# For some uses you can also look into custom vars_plugins to merge on input, even substituting the default ``host_group_vars`` that is in charge of parsing the ``host_vars/`` and ``group_vars/`` directories. Most users of this setting are only interested in inventory scope, but the setting itself affects all sources and makes debugging even harder. +# All playbooks and roles in the official examples repos assume the default for this setting. +# Changing the setting to ``merge`` applies across variable sources, but many sources will internally still overwrite the variables. For example ``include_vars`` will dedupe variables internally before updating Ansible, with 'last defined' overwriting previous definitions in same file. +# The Ansible project recommends you **avoid ``merge`` for new projects.** +# It is the intention of the Ansible developers to eventually deprecate and remove this setting, but it is being kept as some users do heavily rely on it. New projects should **avoid 'merge'**. +;hash_behaviour=replace + +# (pathlist) Comma-separated list of Ansible inventory sources +;inventory=/etc/ansible/hosts + +# (pathspec) Colon-separated paths in which Ansible will search for HttpApi Plugins. +;httpapi_plugins=/home/nikos/.ansible/plugins/httpapi:/usr/share/ansible/plugins/httpapi + +# (float) This sets the interval (in seconds) of Ansible internal processes polling each other. Lower values improve performance with large playbooks at the expense of extra CPU load. Higher values are more suitable for Ansible usage in automation scenarios when UI responsiveness is not required but CPU usage might be a concern. +# The default corresponds to the value hardcoded in Ansible <= 2.1 +;internal_poll_interval=0.001 + +# (pathspec) Colon-separated paths in which Ansible will search for Inventory Plugins. +;inventory_plugins=/home/nikos/.ansible/plugins/inventory:/usr/share/ansible/plugins/inventory + +# (string) This is a developer-specific feature that allows enabling additional Jinja2 extensions. +# See the Jinja2 documentation for details. If you do not know what these do, you probably don't need to change this setting :) +;jinja2_extensions=[] + +# (boolean) This option preserves variable types during template operations. +;jinja2_native=False + +# (boolean) Enables/disables the cleaning up of the temporary files Ansible used to execute the tasks on the remote. +# If this option is enabled it will disable ``ANSIBLE_PIPELINING``. +;keep_remote_files=False + +# (boolean) Controls whether callback plugins are loaded when running /usr/bin/ansible. This may be used to log activity from the command line, send notifications, and so on. Callback plugins are always loaded for ``ansible-playbook``. +;bin_ansible_callbacks=False + +# (tmppath) Temporary directory for Ansible to use on the controller. +local_tmp=/home/nikos/.ansible/tmp + +# (list) List of logger names to filter out of the log file. +;log_filter= + +# (path) File to which Ansible will log on the controller. +# When not set the logging is disabled. +;log_path= + +# (pathspec) Colon-separated paths in which Ansible will search for Lookup Plugins. +;lookup_plugins=/home/nikos/.ansible/plugins/lookup:/usr/share/ansible/plugins/lookup + +# (string) Sets the macro for the 'ansible_managed' variable available for :ref:`ansible_collections.ansible.builtin.template_module` and :ref:`ansible_collections.ansible.windows.win_template_module`. This is only relevant to those two modules. +;ansible_managed=Ansible managed + +# (string) This sets the default arguments to pass to the ``ansible`` adhoc binary if no ``-a`` is specified. +;module_args= + +# (string) Compression scheme to use when transferring Python modules to the target. +;module_compression=ZIP_DEFLATED + +# (string) Module to use with the ``ansible`` AdHoc command, if none is specified via ``-m``. +;module_name=command + +# (pathspec) Colon-separated paths in which Ansible will search for Modules. +;library=/home/nikos/.ansible/plugins/modules:/usr/share/ansible/plugins/modules + +# (pathspec) Colon-separated paths in which Ansible will search for Module utils files, which are shared by modules. +;module_utils=/home/nikos/.ansible/plugins/module_utils:/usr/share/ansible/plugins/module_utils + +# (pathspec) Colon-separated paths in which Ansible will search for Netconf Plugins. +;netconf_plugins=/home/nikos/.ansible/plugins/netconf:/usr/share/ansible/plugins/netconf + +# (boolean) Toggle Ansible's display and logging of task details, mainly used to avoid security disclosures. +;no_log=False + +# (boolean) Toggle Ansible logging to syslog on the target when it executes tasks. On Windows hosts, this will disable a newer style PowerShell modules from writing to the event log. +;no_target_syslog=False + +# (raw) What templating should return as a 'null' value. When not set it will let Jinja2 decide. +;null_representation= + +# (integer) For asynchronous tasks in Ansible (covered in Asynchronous Actions and Polling), this is how often to check back on the status of those tasks when an explicit poll interval is not supplied. The default is a reasonably moderate 15 seconds which is a tradeoff between checking in frequently and providing a quick turnaround when something may have completed. +;poll_interval=15 + +# (path) Option for connections using a certificate or key file to authenticate, rather than an agent or passwords, you can set the default value here to avoid re-specifying ``--private-key`` with every invocation. +;private_key_file= + +# (boolean) By default, imported roles publish their variables to the play and other roles, this setting can avoid that. +# This was introduced as a way to reset role variables to default values if a role is used more than once in a playbook. +# Starting in version '2.17' M(ansible.builtin.include_roles) and M(ansible.builtin.import_roles) can individually override this via the C(public) parameter. +# Included roles only make their variables public at execution, unlike imported roles which happen at playbook compile time. +;private_role_vars=False + +# (integer) Port to use in remote connections, when blank it will use the connection plugin default. +;remote_port= + +# (string) Sets the login user for the target machines +# When blank it uses the connection plugin's default, normally the user currently executing Ansible. +;remote_user= + +# (pathspec) Colon-separated paths in which Ansible will search for Roles. +;roles_path=/home/nikos/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles + +# (string) Set the main callback used to display Ansible output. You can only have one at a time. +# You can have many other callbacks, but just one can be in charge of stdout. +# See :ref:`callback_plugins` for a list of available options. +stdout_callback=minimal + +# (string) Set the default strategy used for plays. +;strategy=linear + +# (pathspec) Colon-separated paths in which Ansible will search for Strategy Plugins. +;strategy_plugins=/home/nikos/.ansible/plugins/strategy:/usr/share/ansible/plugins/strategy + +# (boolean) Toggle the use of "su" for tasks. +;su=False + +# (string) Syslog facility to use when Ansible logs to the remote target. +;syslog_facility=LOG_USER + +# (pathspec) Colon-separated paths in which Ansible will search for Terminal Plugins. +;terminal_plugins=/home/nikos/.ansible/plugins/terminal:/usr/share/ansible/plugins/terminal + +# (pathspec) Colon-separated paths in which Ansible will search for Jinja2 Test Plugins. +;test_plugins=/home/nikos/.ansible/plugins/test:/usr/share/ansible/plugins/test + +# (integer) This is the default timeout for connection plugins to use. +;timeout=10 + +# (string) Can be any connection plugin available to your ansible installation. +# There is also a (DEPRECATED) special 'smart' option, that will toggle between 'ssh' and 'paramiko' depending on controller OS and ssh versions. +;transport=ssh + +# (boolean) When True, this causes ansible templating to fail steps that reference variable names that are likely typoed. +# Otherwise, any '{{ template_expression }}' that contains undefined variables will be rendered in a template or ansible action line exactly as written. +;error_on_undefined_vars=True + +# (pathspec) Colon-separated paths in which Ansible will search for Vars Plugins. +;vars_plugins=/home/nikos/.ansible/plugins/vars:/usr/share/ansible/plugins/vars + +# (string) The vault_id to use for encrypting by default. If multiple vault_ids are provided, this specifies which to use for encryption. The ``--encrypt-vault-id`` CLI option overrides the configured value. +;vault_encrypt_identity= + +# (string) The label to use for the default vault id label in cases where a vault id label is not provided. +;vault_identity=default + +# (list) A list of vault-ids to use by default. Equivalent to multiple ``--vault-id`` args. Vault-ids are tried in order. +;vault_identity_list= + +# (string) If true, decrypting vaults with a vault id will only try the password from the matching vault-id. +;vault_id_match=False + +# (path) The vault password file to use. Equivalent to ``--vault-password-file`` or ``--vault-id``. +# If executable, it will be run and the resulting stdout will be used as the password. +;vault_password_file= + +# (integer) Sets the default verbosity, equivalent to the number of ``-v`` passed in the command line. +;verbosity=0 + +# (boolean) Toggle to control the showing of deprecation warnings +;deprecation_warnings=True + +# (boolean) Toggle to control showing warnings related to running devel. +;devel_warning=True + +# (boolean) Normally ``ansible-playbook`` will print a header for each task that is run. These headers will contain the name: field from the task if you specified one. If you didn't then ``ansible-playbook`` uses the task's action to help you tell which task is presently running. Sometimes you run many of the same action and so you want more information about the task to differentiate it from others of the same action. If you set this variable to True in the config then ``ansible-playbook`` will also include the task's arguments in the header. +# This setting defaults to False because there is a chance that you have sensitive values in your parameters and you do not want those to be printed. +# If you set this to True you should be sure that you have secured your environment's stdout (no one can shoulder surf your screen and you aren't saving stdout to an insecure file) or made sure that all of your playbooks explicitly added the ``no_log: True`` parameter to tasks that have sensitive values :ref:`keep_secret_data` for more information. +;display_args_to_stdout=False + +# (boolean) Toggle to control displaying skipped task/host entries in a task in the default callback. +display_skipped_hosts=False + +# (string) Root docsite URL used to generate docs URLs in warning/error text; must be an absolute URL with a valid scheme and trailing slash. +;docsite_root_url=https://docs.ansible.com/ansible-core/ + +# (pathspec) Colon-separated paths in which Ansible will search for Documentation Fragments Plugins. +;doc_fragment_plugins=/home/nikos/.ansible/plugins/doc_fragments:/usr/share/ansible/plugins/doc_fragments + +# (string) By default, Ansible will issue a warning when a duplicate dict key is encountered in YAML. +# These warnings can be silenced by adjusting this setting to False. +;duplicate_dict_key=warn + +# (string) for the cases in which Ansible needs to return a file within an editor, this chooses the application to use. +;editor=vi + +# (boolean) Whether or not to enable the task debugger, this previously was done as a strategy plugin. +# Now all strategy plugins can inherit this behavior. The debugger defaults to activating when +# a task is failed on unreachable. Use the debugger keyword for more flexibility. +;enable_task_debugger=False + +# (boolean) Toggle to allow missing handlers to become a warning instead of an error when notifying. +;error_on_missing_handler=True + +# (list) Which modules to run during a play's fact gathering stage, using the default of 'smart' will try to figure it out based on connection type. +# If adding your own modules but you still want to use the default Ansible facts, you will want to include 'setup' or corresponding network module to the list (if you add 'smart', Ansible will also figure it out). +# This does not affect explicit calls to the 'setup' module, but does always affect the 'gather_facts' action (implicit or explicit). +;facts_modules=smart + +# (boolean) Set this to "False" if you want to avoid host key checking by the underlying connection plugin Ansible uses to connect to the host. +# Please read the documentation of the specific connection plugin used for details. +host_key_checking=False + +# (boolean) Facts are available inside the `ansible_facts` variable, this setting also pushes them as their own vars in the main namespace. +# Unlike inside the `ansible_facts` dictionary where the prefix `ansible_` is removed from fact names, these will have the exact names that are returned by the module. +;inject_facts_as_vars=True + +# (string) Path to the Python interpreter to be used for module execution on remote targets, or an automatic discovery mode. Supported discovery modes are ``auto`` (the default), ``auto_silent``, ``auto_legacy``, and ``auto_legacy_silent``. All discovery modes employ a lookup table to use the included system Python (on distributions known to include one), falling back to a fixed ordered list of well-known Python interpreter locations if a platform-specific default is not available. The fallback behavior will issue a warning that the interpreter should be set explicitly (since interpreters installed later may change which one is used). This warning behavior can be disabled by setting ``auto_silent`` or ``auto_legacy_silent``. The value of ``auto_legacy`` provides all the same behavior, but for backward-compatibility with older Ansible releases that always defaulted to ``/usr/bin/python``, will use that interpreter if present. +;interpreter_python=auto + +# (boolean) If 'false', invalid attributes for a task will result in warnings instead of errors. +;invalid_task_attribute_failed=True + +# (boolean) By default, Ansible will issue a warning when there are no hosts in the inventory. +# These warnings can be silenced by adjusting this setting to False. +;localhost_warning=True + +# (int) This will set log verbosity if higher than the normal display verbosity, otherwise it will match that. +;log_verbosity= + +# (int) Maximum size of files to be considered for diff display. +;max_diff_size=104448 + +# (list) List of extensions to ignore when looking for modules to load. +# This is for rejecting script and binary module fallback extensions. +;module_ignore_exts=.pyc, .pyo, .swp, .bak, ~, .rpm, .md, .txt, .rst, .yaml, .yml, .ini + +# (bool) Enables whether module responses are evaluated for containing non-UTF-8 data. +# Disabling this may result in unexpected behavior. +# Only ansible-core should evaluate this configuration. +;module_strict_utf8_response=True + +# (list) TODO: write it +;network_group_modules=eos, nxos, ios, iosxr, junos, enos, ce, vyos, sros, dellos9, dellos10, dellos6, asa, aruba, aireos, bigip, ironware, onyx, netconf, exos, voss, slxos + +# (boolean) Previously Ansible would only clear some of the plugin loading caches when loading new roles, this led to some behaviors in which a plugin loaded in previous plays would be unexpectedly 'sticky'. This setting allows the user to return to that behavior. +;old_plugin_cache_clear=False + +# (string) for the cases in which Ansible needs to return output in a pageable fashion, this chooses the application to use. +;pager=less + +# (path) A number of non-playbook CLIs have a ``--playbook-dir`` argument; this sets the default value for it. +;playbook_dir= + +# (string) This sets which playbook dirs will be used as a root to process vars plugins, which includes finding host_vars/group_vars. +;playbook_vars_root=top + +# (path) A path to configuration for filtering which plugins installed on the system are allowed to be used. +# See :ref:`plugin_filtering_config` for details of the filter file's format. +# The default is /etc/ansible/plugin_filters.yml +;plugin_filters_cfg= + +# (string) Attempts to set RLIMIT_NOFILE soft limit to the specified value when executing Python modules (can speed up subprocess usage on Python 2.x. See https://bugs.python.org/issue11284). The value will be limited by the existing hard limit. Default value of 0 does not attempt to adjust existing system-defined limits. +;python_module_rlimit_nofile=0 + +# (bool) This controls whether a failed Ansible playbook should create a .retry file. +retry_files_enabled=False + +# (path) This sets the path in which Ansible will save .retry files when a playbook fails and retry files are enabled. +# This file will be overwritten after each run with the list of failed hosts from all plays. +;retry_files_save_path= + +# (str) This setting can be used to optimize vars_plugin usage depending on the user's inventory size and play selection. +;run_vars_plugins=demand + +# (bool) This adds the custom stats set via the set_stats plugin to the default output. +;show_custom_stats=False + +# (string) Action to take when a module parameter value is converted to a string (this does not affect variables). For string parameters, values such as '1.00', "['a', 'b',]", and 'yes', 'y', etc. will be converted by the YAML parser unless fully quoted. +# Valid options are 'error', 'warn', and 'ignore'. +# Since 2.8, this option defaults to 'warn' but will change to 'error' in 2.12. +;string_conversion_action=warn + +# (boolean) Allows disabling of warnings related to potential issues on the system running Ansible itself (not on the managed hosts). +# These may include warnings about third-party packages or other conditions that should be resolved if possible. +;system_warnings=True + +# (string) A string to insert into target logging for tracking purposes +;target_log_info= + +# (boolean) This option defines whether the task debugger will be invoked on a failed task when ignore_errors=True is specified. +# True specifies that the debugger will honor ignore_errors, and False will not honor ignore_errors. +;task_debugger_ignore_errors=True + +# (integer) Set the maximum time (in seconds) for a task action to execute in. +# Timeout runs independently from templating or looping. It applies per each attempt of executing the task's action and remains unchanged by the total time spent on a task. +# When the action execution exceeds the timeout, Ansible interrupts the process. This is registered as a failure due to outside circumstances, not a task failure, to receive appropriate response and recovery process. +# If set to 0 (the default) there is no timeout. +;task_timeout=0 + +# (string) Make ansible transform invalid characters in group names supplied by inventory sources. +;force_valid_group_names=never + +# (boolean) Toggles the use of persistence for connections. +;use_persistent_connections=False + +# (bool) A toggle to disable validating a collection's 'metadata' entry for a module_defaults action group. Metadata containing unexpected fields or value types will produce a warning when this is True. +;validate_action_group_metadata=True + +# (list) Accept list for variable plugins that require it. +;vars_plugins_enabled=host_group_vars + +# (list) Allows to change the group variable precedence merge order. +;precedence=all_inventory, groups_inventory, all_plugins_inventory, all_plugins_play, groups_plugins_inventory, groups_plugins_play + +# (string) The salt to use for the vault encryption. If it is not provided, a random salt will be used. +;vault_encrypt_salt= + +# (bool) Force 'verbose' option to use stderr instead of stdout +;verbose_to_stderr=False + +# (integer) For asynchronous tasks in Ansible (covered in Asynchronous Actions and Polling), this is how long, in seconds, to wait for the task spawned by Ansible to connect back to the named pipe used on Windows systems. The default is 5 seconds. This can be too low on slower systems, or systems under heavy load. +# This is not the total time an async command can run for, but is a separate timeout to wait for an async command to start. The task will only start to be timed against its async_timeout once it has connected to the pipe, so the overall maximum duration the task can take will be extended by the amount specified here. +;win_async_startup_timeout=5 + +# (list) Check all of these extensions when looking for 'variable' files which should be YAML or JSON or vaulted versions of these. +# This affects vars_files, include_vars, inventory and vars plugins among others. +;yaml_valid_extensions=.yml, .yaml, .json + + +[privilege_escalation] +# (boolean) Display an agnostic become prompt instead of displaying a prompt containing the command line supplied become method. +;agnostic_become_prompt=True + +# (boolean) When ``False``(default), Ansible will skip using become if the remote user is the same as the become user, as this is normally a redundant operation. In other words root sudo to root. +# If ``True``, this forces Ansible to use the become plugin anyways as there are cases in which this is needed. +;become_allow_same_user=False + +# (boolean) Toggles the use of privilege escalation, allowing you to 'become' another user after login. +;become=False + +# (boolean) Toggle to prompt for privilege escalation password. +;become_ask_pass=False + +# (string) executable to use for privilege escalation, otherwise Ansible will depend on PATH. +;become_exe= + +# (string) Flags to pass to the privilege escalation executable. +;become_flags= + +# (string) Privilege escalation method to use when `become` is enabled. +;become_method=sudo + +# (string) The user your login/remote user 'becomes' when using privilege escalation, most systems will use 'root' when no user is specified. +;become_user=root + + +[persistent_connection] +# (path) Specify where to look for the ansible-connection script. This location will be checked before searching $PATH. +# If null, ansible will start with the same directory as the ansible script. +;ansible_connection_path= + +# (int) This controls the amount of time to wait for a response from a remote device before timing out a persistent connection. +;command_timeout=30 + +# (integer) This controls the retry timeout for persistent connection to connect to the local domain socket. +;connect_retry_timeout=15 + +# (integer) This controls how long the persistent connection will remain idle before it is destroyed. +;connect_timeout=30 + +# (path) Path to the socket to be used by the connection persistence system. +;control_path_dir=/home/nikos/.ansible/pc + + +[connection] +# (boolean) This is a global option, each connection plugin can override either by having more specific options or not supporting pipelining at all. +# Pipelining, if supported by the connection plugin, reduces the number of network operations required to execute a module on the remote server, by executing many Ansible modules without actual file transfer. +# It can result in a very significant performance improvement when enabled. +# However this conflicts with privilege escalation (become). For example, when using 'sudo:' operations you must first disable 'requiretty' in /etc/sudoers on all managed hosts, which is why it is disabled by default. +# This setting will be disabled if ``ANSIBLE_KEEP_REMOTE_FILES`` is enabled. +;pipelining=False + + +[colors] +# (string) Defines the color to use on 'Changed' task status. +;changed=yellow + +# (string) Defines the default color to use for ansible-console. +;console_prompt=white + +# (string) Defines the color to use when emitting debug messages. +;debug=dark gray + +# (string) Defines the color to use when emitting deprecation messages. +;deprecate=purple + +# (string) Defines the color to use when showing added lines in diffs. +;diff_add=green + +# (string) Defines the color to use when showing diffs. +;diff_lines=cyan + +# (string) Defines the color to use when showing removed lines in diffs. +;diff_remove=red + +# (string) Defines the color to use when emitting a constant in the ansible-doc output. +;doc_constant=dark gray + +# (string) Defines the color to use when emitting a deprecated value in the ansible-doc output. +;doc_deprecated=magenta + +# (string) Defines the color to use when emitting a link in the ansible-doc output. +;doc_link=cyan + +# (string) Defines the color to use when emitting a module name in the ansible-doc output. +;doc_module=yellow + +# (string) Defines the color to use when emitting a plugin name in the ansible-doc output. +;doc_plugin=yellow + +# (string) Defines the color to use when emitting cross-reference in the ansible-doc output. +;doc_reference=magenta + +# (string) Defines the color to use when emitting error messages. +;error=red + +# (string) Defines the color to use for highlighting. +;highlight=white + +# (string) Defines the color to use when showing 'Included' task status. +;included=cyan + +# (string) Defines the color to use when showing 'OK' task status. +;ok=green + +# (string) Defines the color to use when showing 'Skipped' task status. +;skip=cyan + +# (string) Defines the color to use on 'Unreachable' status. +;unreachable=bright red + +# (string) Defines the color to use when emitting verbose messages. In other words, those that show with '-v's. +;verbose=blue + +# (string) Defines the color to use when emitting warning messages. +;warn=bright purple + + +[selinux] +# (boolean) This setting causes libvirt to connect to LXC containers by passing ``--noseclabel`` parameter to ``virsh`` command. This is necessary when running on systems which do not have SELinux. +;libvirt_lxc_noseclabel=False + +# (list) Some filesystems do not support safe operations and/or return inconsistent errors, this setting makes Ansible 'tolerate' those in the list without causing fatal errors. +# Data corruption may occur and writes are not always verified when a filesystem is in the list. +;special_context_filesystems=fuse, nfs, vboxsf, ramfs, 9p, vfat + + +[diff] +# (bool) Configuration toggle to tell modules to show differences when in 'changed' status, equivalent to ``--diff``. +;always=False + +# (integer) Number of lines of context to show when displaying the differences between files. +;context=3 + + +[galaxy] +# (path) The directory that stores cached responses from a Galaxy server. +# This is only used by the ``ansible-galaxy collection install`` and ``download`` commands. +# Cache files inside this dir will be ignored if they are world writable. +;cache_dir=/home/nikos/.ansible/galaxy_cache + +# (bool) whether ``ansible-galaxy collection install`` should warn about ``--collections-path`` missing from configured :ref:`collections_paths`. +;collections_path_warning=True + +# (path) Collection skeleton directory to use as a template for the ``init`` action in ``ansible-galaxy collection``, same as ``--collection-skeleton``. +;collection_skeleton= + +# (list) patterns of files to ignore inside a Galaxy collection skeleton directory. +;collection_skeleton_ignore=^.git$, ^.*/.git_keep$ + +# (bool) Disable GPG signature verification during collection installation. +;disable_gpg_verify=False + +# (bool) Some steps in ``ansible-galaxy`` display a progress wheel which can cause issues on certain displays or when outputting the stdout to a file. +# This config option controls whether the display wheel is shown or not. +# The default is to show the display wheel if stdout has a tty. +;display_progress= + +# (path) Configure the keyring used for GPG signature verification during collection installation and verification. +;gpg_keyring= + +# (boolean) If set to yes, ansible-galaxy will not validate TLS certificates. This can be useful for testing against a server with a self-signed certificate. +;ignore_certs= + +# (list) A list of GPG status codes to ignore during GPG signature verification. See L(https://github.com/gpg/gnupg/blob/master/doc/DETAILS#general-status-codes) for status code descriptions. +# If fewer signatures successfully verify the collection than `GALAXY_REQUIRED_VALID_SIGNATURE_COUNT`, signature verification will fail even if all error codes are ignored. +;ignore_signature_status_codes= + +# (str) The number of signatures that must be successful during GPG signature verification while installing or verifying collections. +# This should be a positive integer or all to indicate all signatures must successfully validate the collection. +# Prepend + to the value to fail if no valid signatures are found for the collection. +;required_valid_signature_count=1 + +# (path) Role skeleton directory to use as a template for the ``init`` action in ``ansible-galaxy``/``ansible-galaxy role``, same as ``--role-skeleton``. +;role_skeleton= + +# (list) patterns of files to ignore inside a Galaxy role or collection skeleton directory. +;role_skeleton_ignore=^.git$, ^.*/.git_keep$ + +# (string) URL to prepend when roles don't specify the full URI, assume they are referencing this server as the source. +;server=https://galaxy.ansible.com + +# (list) A list of Galaxy servers to use when installing a collection. +# The value corresponds to the config ini header ``[galaxy_server.{{item}}]`` which defines the server details. +# See :ref:`galaxy_server_config` for more details on how to define a Galaxy server. +# The order of servers in this list is used as the order in which a collection is resolved. +# Setting this config option will ignore the :ref:`galaxy_server` config option. +;server_list= + +# (int) The default timeout for Galaxy API calls. Galaxy servers that don't configure a specific timeout will fall back to this value. +;server_timeout=60 + +# (path) Local path to galaxy access token file +;token_path=/home/nikos/.ansible/galaxy_token + + +[inventory] +# (string) This setting changes the behaviour of mismatched host patterns, it allows you to force a fatal error, a warning or just ignore it. +;host_pattern_mismatch=warning + +# (boolean) If 'true', it is a fatal error when any given inventory source cannot be successfully parsed by any available inventory plugin; otherwise, this situation only attracts a warning. + +;any_unparsed_is_failed=False + +# (bool) Toggle to turn on inventory caching. +# This setting has been moved to the individual inventory plugins as a plugin option :ref:`inventory_plugins`. +# The existing configuration settings are still accepted with the inventory plugin adding additional options from inventory configuration. +# This message will be removed in 2.16. +;cache=False + +# (string) The plugin for caching inventory. +# This setting has been moved to the individual inventory plugins as a plugin option :ref:`inventory_plugins`. +# The existing configuration settings are still accepted with the inventory plugin adding additional options from inventory and fact cache configuration. +# This message will be removed in 2.16. +;cache_plugin= + +# (string) The inventory cache connection. +# This setting has been moved to the individual inventory plugins as a plugin option :ref:`inventory_plugins`. +# The existing configuration settings are still accepted with the inventory plugin adding additional options from inventory and fact cache configuration. +# This message will be removed in 2.16. +;cache_connection= + +# (string) The table prefix for the cache plugin. +# This setting has been moved to the individual inventory plugins as a plugin option :ref:`inventory_plugins`. +# The existing configuration settings are still accepted with the inventory plugin adding additional options from inventory and fact cache configuration. +# This message will be removed in 2.16. +;cache_prefix=ansible_inventory_ + +# (string) Expiration timeout for the inventory cache plugin data. +# This setting has been moved to the individual inventory plugins as a plugin option :ref:`inventory_plugins`. +# The existing configuration settings are still accepted with the inventory plugin adding additional options from inventory and fact cache configuration. +# This message will be removed in 2.16. +;cache_timeout=3600 + +# (list) List of enabled inventory plugins, it also determines the order in which they are used. +;enable_plugins=host_list, script, auto, yaml, ini, toml + +# (bool) Controls if ansible-inventory will accurately reflect Ansible's view into inventory or its optimized for exporting. +;export=False + +# (list) List of extensions to ignore when using a directory as an inventory source. +;ignore_extensions=.pyc, .pyo, .swp, .bak, ~, .rpm, .md, .txt, .rst, .orig, .ini, .cfg, .retry + +# (list) List of patterns to ignore when using a directory as an inventory source. +;ignore_patterns= + +# (bool) If 'true' it is a fatal error if every single potential inventory source fails to parse, otherwise, this situation will only attract a warning. + +;unparsed_is_failed=False + +# (boolean) By default, Ansible will issue a warning when no inventory was loaded and notes that it will use an implicit localhost-only inventory. +# These warnings can be silenced by adjusting this setting to False. +;inventory_unparsed_warning=True + + +[netconf_connection] +# (string) This variable is used to enable bastion/jump host with netconf connection. If set to True the bastion/jump host ssh settings should be present in ~/.ssh/config file, alternatively it can be set to custom ssh configuration file path to read the bastion/jump host settings. +;ssh_config= + + +[paramiko_connection] +# (boolean) TODO: write it +;host_key_auto_add=False + +# (boolean) TODO: write it +;look_for_keys=True + + +[jinja2] +# (list) This list of filters avoids 'type conversion' when templating variables. +# Useful when you want to avoid conversion into lists or dictionaries for JSON strings, for example. +;dont_type_filters=string, to_json, to_nice_json, to_yaml, to_nice_yaml, ppretty, json + + +[tags] +# (list) default list of tags to run in your plays, Skip Tags has precedence. +;run= + +# (list) default list of tags to skip in your plays, has precedence over Run Tags +;skip= + diff --git a/ansible_verbose.cfg b/src/dmtri/data/ansible_verbose.cfg similarity index 95% rename from ansible_verbose.cfg rename to src/dmtri/data/ansible_verbose.cfg index 87a5256..6ce59d5 100644 --- a/ansible_verbose.cfg +++ b/src/dmtri/data/ansible_verbose.cfg @@ -32,10 +32,10 @@ ;become_password_file= # (pathspec) Colon-separated paths in which Ansible will search for Become Plugins. -;become_plugins=/home/nsokos/.ansible/plugins/become:/usr/share/ansible/plugins/become +;become_plugins=/home/nikos/.ansible/plugins/become:/usr/share/ansible/plugins/become # (string) Chooses which cache plugin to use, the default 'memory' is ephemeral. -;fact_caching=memory +fact_caching=memory # (string) Defines connection or path information for the cache plugin. ;fact_caching_connection= @@ -54,7 +54,7 @@ # (pathspec) Colon-separated paths in which Ansible will search for collections content. Collections must be in nested *subdirectories*, not directly in these directories. For example, if ``COLLECTIONS_PATHS`` includes ``'{{ ANSIBLE_HOME ~ "/collections" }}'``, and you want to add ``my.collection`` to that directory, it must be saved as ``'{{ ANSIBLE_HOME} ~ "/collections/ansible_collections/my/collection" }}'``. -;collections_path=/home/nsokos/.ansible/collections:/usr/share/ansible/collections +;collections_path=/home/nikos/.ansible/collections:/usr/share/ansible/collections # (boolean) A boolean to enable or disable scanning the sys.path for installed collections. ;collections_scan_sys_path=True @@ -63,7 +63,7 @@ ;connection_password_file= # (pathspec) Colon-separated paths in which Ansible will search for Action Plugins. -;action_plugins=/home/nsokos/.ansible/plugins/action:/usr/share/ansible/plugins/action +;action_plugins=/home/nikos/.ansible/plugins/action:/usr/share/ansible/plugins/action # (boolean) When enabled, this option allows lookup plugins (whether used in variables as ``{{lookup('foo')}}`` or as a loop as with_foo) to return data that is not marked 'unsafe'. # By default, such data is marked as unsafe to prevent the templating engine from evaluating any jinja2 templating language, as this could represent a security risk. This option is provided to allow for backward compatibility, however, users should first consider adding allow_unsafe=True to any lookups that may be expected to contain data that may be run through the templating engine late. @@ -76,16 +76,16 @@ ;ask_vault_pass=False # (pathspec) Colon-separated paths in which Ansible will search for Cache Plugins. -;cache_plugins=/home/nsokos/.ansible/plugins/cache:/usr/share/ansible/plugins/cache +;cache_plugins=/home/nikos/.ansible/plugins/cache:/usr/share/ansible/plugins/cache # (pathspec) Colon-separated paths in which Ansible will search for Callback Plugins. -;callback_plugins=/home/nsokos/.ansible/plugins/callback:/usr/share/ansible/plugins/callback +;callback_plugins=/home/nikos/.ansible/plugins/callback:/usr/share/ansible/plugins/callback # (pathspec) Colon-separated paths in which Ansible will search for Cliconf Plugins. -;cliconf_plugins=/home/nsokos/.ansible/plugins/cliconf:/usr/share/ansible/plugins/cliconf +;cliconf_plugins=/home/nikos/.ansible/plugins/cliconf:/usr/share/ansible/plugins/cliconf # (pathspec) Colon-separated paths in which Ansible will search for Connection Plugins. -;connection_plugins=/home/nsokos/.ansible/plugins/connection:/usr/share/ansible/plugins/connection +;connection_plugins=/home/nikos/.ansible/plugins/connection:/usr/share/ansible/plugins/connection # (boolean) Toggles debug output in Ansible. This is *very* verbose and can hinder multiprocessing. Debug output can also include secret information despite no_log settings being enabled, which means debug mode should not be used in production. ;debug=False @@ -94,7 +94,7 @@ ;executable=/bin/sh # (pathspec) Colon-separated paths in which Ansible will search for Jinja2 Filter Plugins. -;filter_plugins=/home/nsokos/.ansible/plugins/filter:/usr/share/ansible/plugins/filter +;filter_plugins=/home/nikos/.ansible/plugins/filter:/usr/share/ansible/plugins/filter # (boolean) This option controls if notified handlers run on a host even if a failure occurs on that host. # When false, the handlers will not run if a failure has occurred on a host. @@ -123,14 +123,14 @@ ;inventory=/etc/ansible/hosts # (pathspec) Colon-separated paths in which Ansible will search for HttpApi Plugins. -;httpapi_plugins=/home/nsokos/.ansible/plugins/httpapi:/usr/share/ansible/plugins/httpapi +;httpapi_plugins=/home/nikos/.ansible/plugins/httpapi:/usr/share/ansible/plugins/httpapi # (float) This sets the interval (in seconds) of Ansible internal processes polling each other. Lower values improve performance with large playbooks at the expense of extra CPU load. Higher values are more suitable for Ansible usage in automation scenarios when UI responsiveness is not required but CPU usage might be a concern. # The default corresponds to the value hardcoded in Ansible <= 2.1 ;internal_poll_interval=0.001 # (pathspec) Colon-separated paths in which Ansible will search for Inventory Plugins. -;inventory_plugins=/home/nsokos/.ansible/plugins/inventory:/usr/share/ansible/plugins/inventory +;inventory_plugins=/home/nikos/.ansible/plugins/inventory:/usr/share/ansible/plugins/inventory # (string) This is a developer-specific feature that allows enabling additional Jinja2 extensions. # See the Jinja2 documentation for details. If you do not know what these do, you probably don't need to change this setting :) @@ -147,7 +147,7 @@ ;bin_ansible_callbacks=False # (tmppath) Temporary directory for Ansible to use on the controller. -;local_tmp=/home/nsokos/.ansible/tmp +local_tmp=/home/nikos/.ansible/tmp # (list) List of logger names to filter out of the log file. ;log_filter= @@ -157,7 +157,7 @@ ;log_path= # (pathspec) Colon-separated paths in which Ansible will search for Lookup Plugins. -;lookup_plugins=/home/nsokos/.ansible/plugins/lookup:/usr/share/ansible/plugins/lookup +;lookup_plugins=/home/nikos/.ansible/plugins/lookup:/usr/share/ansible/plugins/lookup # (string) Sets the macro for the 'ansible_managed' variable available for :ref:`ansible_collections.ansible.builtin.template_module` and :ref:`ansible_collections.ansible.windows.win_template_module`. This is only relevant to those two modules. ;ansible_managed=Ansible managed @@ -172,13 +172,13 @@ ;module_name=command # (pathspec) Colon-separated paths in which Ansible will search for Modules. -;library=/home/nsokos/.ansible/plugins/modules:/usr/share/ansible/plugins/modules +;library=/home/nikos/.ansible/plugins/modules:/usr/share/ansible/plugins/modules # (pathspec) Colon-separated paths in which Ansible will search for Module utils files, which are shared by modules. -;module_utils=/home/nsokos/.ansible/plugins/module_utils:/usr/share/ansible/plugins/module_utils +;module_utils=/home/nikos/.ansible/plugins/module_utils:/usr/share/ansible/plugins/module_utils # (pathspec) Colon-separated paths in which Ansible will search for Netconf Plugins. -;netconf_plugins=/home/nsokos/.ansible/plugins/netconf:/usr/share/ansible/plugins/netconf +;netconf_plugins=/home/nikos/.ansible/plugins/netconf:/usr/share/ansible/plugins/netconf # (boolean) Toggle Ansible's display and logging of task details, mainly used to avoid security disclosures. ;no_log=False @@ -209,7 +209,7 @@ ;remote_user= # (pathspec) Colon-separated paths in which Ansible will search for Roles. -;roles_path=/home/nsokos/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles +;roles_path=/home/nikos/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles # (string) Set the main callback used to display Ansible output. You can only have one at a time. # You can have many other callbacks, but just one can be in charge of stdout. @@ -220,7 +220,7 @@ ;strategy=linear # (pathspec) Colon-separated paths in which Ansible will search for Strategy Plugins. -;strategy_plugins=/home/nsokos/.ansible/plugins/strategy:/usr/share/ansible/plugins/strategy +;strategy_plugins=/home/nikos/.ansible/plugins/strategy:/usr/share/ansible/plugins/strategy # (boolean) Toggle the use of "su" for tasks. ;su=False @@ -229,10 +229,10 @@ ;syslog_facility=LOG_USER # (pathspec) Colon-separated paths in which Ansible will search for Terminal Plugins. -;terminal_plugins=/home/nsokos/.ansible/plugins/terminal:/usr/share/ansible/plugins/terminal +;terminal_plugins=/home/nikos/.ansible/plugins/terminal:/usr/share/ansible/plugins/terminal # (pathspec) Colon-separated paths in which Ansible will search for Jinja2 Test Plugins. -;test_plugins=/home/nsokos/.ansible/plugins/test:/usr/share/ansible/plugins/test +;test_plugins=/home/nikos/.ansible/plugins/test:/usr/share/ansible/plugins/test # (integer) This is the default timeout for connection plugins to use. ;timeout=10 @@ -246,7 +246,7 @@ ;error_on_undefined_vars=True # (pathspec) Colon-separated paths in which Ansible will search for Vars Plugins. -;vars_plugins=/home/nsokos/.ansible/plugins/vars:/usr/share/ansible/plugins/vars +;vars_plugins=/home/nikos/.ansible/plugins/vars:/usr/share/ansible/plugins/vars # (string) The vault_id to use for encrypting by default. If multiple vault_ids are provided, this specifies which to use for encryption. The ``--encrypt-vault-id`` CLI option overrides the configured value. ;vault_encrypt_identity= @@ -285,7 +285,7 @@ ;docsite_root_url=https://docs.ansible.com/ansible-core/ # (pathspec) Colon-separated paths in which Ansible will search for Documentation Fragments Plugins. -;doc_fragment_plugins=/home/nsokos/.ansible/plugins/doc_fragments:/usr/share/ansible/plugins/doc_fragments +;doc_fragment_plugins=/home/nikos/.ansible/plugins/doc_fragments:/usr/share/ansible/plugins/doc_fragments # (string) By default, Ansible will issue a warning when a duplicate dict key is encountered in YAML. # These warnings can be silenced by adjusting this setting to False. @@ -470,7 +470,7 @@ ;connect_timeout=30 # (path) Path to the socket to be used by the connection persistence system. -;control_path_dir=/home/nsokos/.ansible/pc +;control_path_dir=/home/nikos/.ansible/pc [connection] @@ -568,7 +568,7 @@ # (path) The directory that stores cached responses from a Galaxy server. # This is only used by the ``ansible-galaxy collection install`` and ``download`` commands. # Cache files inside this dir will be ignored if they are world writable. -;cache_dir=/home/nsokos/.ansible/galaxy_cache +;cache_dir=/home/nikos/.ansible/galaxy_cache # (bool) whether ``ansible-galaxy collection install`` should warn about ``--collections-path`` missing from configured :ref:`collections_paths`. ;collections_path_warning=True @@ -622,7 +622,7 @@ ;server_timeout=60 # (path) Local path to galaxy access token file -;token_path=/home/nsokos/.ansible/galaxy_token +;token_path=/home/nikos/.ansible/galaxy_token [inventory] diff --git a/src/inventory/hosts.ini.example b/src/dmtri/data/inventory/hosts.ini.example similarity index 97% rename from src/inventory/hosts.ini.example rename to src/dmtri/data/inventory/hosts.ini.example index 5980f37..f740540 100644 --- a/src/inventory/hosts.ini.example +++ b/src/dmtri/data/inventory/hosts.ini.example @@ -1,23 +1,23 @@ -# hosts.ini.example -# Replace placeholders with your actual host details - -[seedpsd_nodes] - ansible_user= ansible_ssh_pass="" ansible_connection=ssh ansible_become=false ansible_ssh_common_args='-o StrictHostKeyChecking=no' - -[seedpsd_nodes:vars] -seedpsd_dir=/home//seedpsd - -[wf_catalogue] - ansible_user= ansible_ssh_pass="" ansible_connection=ssh ansible_become=false ansible_ssh_common_args='-o StrictHostKeyChecking=no' - -[wf_catalogue:vars] -collector_dir=/home//Programs/wfcatalogue/wfcatalog/collector2 - -[ws_availability] - ansible_user= ansible_ssh_pass="" ansible_connection=ssh ansible_become=false ansible_ssh_common_args='-o StrictHostKeyChecking=no' - -[ws_availability:vars] -availability_dir=/home//Programs/ws-availability/views -mongo_user= -mongo_pass= -mongo_authdb= +# hosts.ini.example +# Replace placeholders with your actual host details + +[seedpsd_nodes] + ansible_user= ansible_ssh_pass="" ansible_connection=ssh ansible_become=false ansible_ssh_common_args='-o StrictHostKeyChecking=no' + +[seedpsd_nodes:vars] +seedpsd_dir=/home//seedpsd + +[wf_catalogue] + ansible_user= ansible_ssh_pass="" ansible_connection=ssh ansible_become=false ansible_ssh_common_args='-o StrictHostKeyChecking=no' + +[wf_catalogue:vars] +collector_dir=/home//Programs/wfcatalogue/wfcatalog/collector2 + +[ws_availability] + ansible_user= ansible_ssh_pass="" ansible_connection=ssh ansible_become=false ansible_ssh_common_args='-o StrictHostKeyChecking=no' + +[ws_availability:vars] +availability_dir=/home//Programs/ws-availability/views +mongo_user= +mongo_pass= +mongo_authdb= diff --git a/src/playbooks/clean/availability_clean.yml b/src/dmtri/data/playbooks/clean/availability_clean.yml similarity index 100% rename from src/playbooks/clean/availability_clean.yml rename to src/dmtri/data/playbooks/clean/availability_clean.yml diff --git a/src/playbooks/clean/seedpsd_clean.yml b/src/dmtri/data/playbooks/clean/seedpsd_clean.yml similarity index 100% rename from src/playbooks/clean/seedpsd_clean.yml rename to src/dmtri/data/playbooks/clean/seedpsd_clean.yml diff --git a/src/dmtri/data/playbooks/clean/wfcatalog_clean.yml b/src/dmtri/data/playbooks/clean/wfcatalog_clean.yml new file mode 100644 index 0000000..278096b --- /dev/null +++ b/src/dmtri/data/playbooks/clean/wfcatalog_clean.yml @@ -0,0 +1,304 @@ +- name: Clean WFCatalog metrics for a given network/station/time range + hosts: wf_catalogue + gather_facts: no + + vars: + venv_python: ".env/bin/python3.11" + collector_script: "WFCatalogCollector.py" + config_path: "{{ collector_dir }}/config.json" + archive_root: "/home/sysop/iso_sds" + + tasks: + + - name: Ensure required variables are provided + fail: + msg: "You must pass 'network', 'station', 'starttime', and 'endtime' via -e" + when: network is not defined or station is not defined or starttime is not defined or endtime is not defined + + - name: Set defaults for optional variables + set_fact: + file_suffix: 'D' + network: "{{ network | default('*') }}" + station: "{{ station | default('*') }}" + location: "{{ location | default('*') }}" + channel: "{{ channel | default('*') }}" + + - name: Compute basic derived values + set_fact: + start_date: "{{ starttime | regex_replace('T.*$', '') }}" + end_date: "{{ endtime | regex_replace('T.*$', '') }}" + + - name: Compute secondary derived values + set_fact: + log_path: "{{ collector_dir }}/log/WFCatalog-clean-{{ network }}-{{ station }}-{{ start_date }}-{{ end_date }}.log" + year_range: >- + {{ range( + (starttime | regex_replace('T.*$', '') | to_datetime('%Y-%m-%d')).year, + (endtime | regex_replace('T.*$', '') | to_datetime('%Y-%m-%d')).year + 1 + ) | list }} + + - name: "DISPLAY CLEAN PARAMETERS" + debug: + msg: + - "========================================" + - " WFCatalog Clean — Parameters" + - "========================================" + - " Host : {{ inventory_hostname }}" + - " Network : {{ network }}" + - " Station : {{ station }}" + - " Location : {{ location }}" + - " Channel : {{ channel }}" + - " Start date : {{ start_date }}" + - " End date : {{ end_date }}" + - " Year range : {{ year_range }}" + - " Log file : {{ log_path }}" + - "========================================" + + # --------------------------------------------------------------- + # FILE DISCOVERY + # --------------------------------------------------------------- + + - name: Generate list of -name patterns for DOY matching across years + shell: | + start="{{ start_date }}" + end="{{ end_date }}" + tmp_list="" + + start_epoch=$(date -d "$start" +%s) + end_epoch=$(date -d "$end" +%s) + days_diff=$(( (end_epoch - start_epoch) / 86400 )) + + for i in $(seq 0 $days_diff); do + current_date=$(date -d "@$((start_epoch + i * 86400))" +%Y.%j) + year=$(echo "$current_date" | cut -d. -f1) + doy=$(echo "$current_date" | cut -d. -f2) + tmp_list="$tmp_list -name '*.${year}.${doy}' -o" + done + + echo "${tmp_list% -o}" + args: + executable: /bin/bash + register: name_patterns + changed_when: false + + - name: Set final find arguments + set_fact: + find_args: "{{ name_patterns.stdout }}" + + - name: Run find in correct SDS folders per year + shell: > + find {{ archive_root }}/{{ item }}/{ {{ network }} }/{ {{ station }} }/{ {{ channel }}.{{ file_suffix }} }/ -type f \( {{ find_args }} \) + loop: "{{ year_range }}" + args: + executable: /bin/bash + register: find_results + changed_when: false + ignore_errors: true + + - name: Collect found SDS file paths + set_fact: + all_files: >- + {{ find_results.results | map(attribute='stdout_lines') | flatten | list }} + + - name: "DISPLAY FILE DISCOVERY RESULTS" + debug: + msg: + - "========================================" + - " WFCatalog Clean — File Discovery" + - "========================================" + - " Total files found : {{ all_files | length }}" + - " First 10 files :" + - "{{ all_files[:10] | join('\n') }}" + - " ({{ [all_files | length - 10, 0] | max }} more not shown)" + - "========================================" + + - name: Abort if no files found + fail: + msg: "No SDS files found for {{ network }}.{{ station }}.{{ location }}.{{ channel }} between {{ start_date }} and {{ end_date }}" + when: all_files | length == 0 + + # --------------------------------------------------------------- + # CONFIG PATCH + # --------------------------------------------------------------- + + - name: Check if backup exists + stat: + path: "{{ config_path }}.back" + register: config_backup + + - name: Backup original config.json + copy: + src: "{{ config_path }}" + dest: "{{ config_path }}.back" + remote_src: true + when: not config_backup.stat.exists + + - name: Read config.json before patch + slurp: + src: "{{ config_path }}" + register: config_before + + - name: "DISPLAY WHITE FILTER BEFORE PATCH" + debug: + msg: + - "========================================" + - " WFCatalog Clean — config.json BEFORE patch" + - "========================================" + - "{{ (config_before.content | b64decode | from_json).FILTERS }}" + - "========================================" + + - name: Update WHITE filter in config.json + shell: | + # Construct a JSON array of filters + NETS="{{ network }}" + STAS="{{ station }}" + LOCS="{{ location }}" + CHAS="{{ channel }}" + + if [[ "$NETS" == *","* || "$STAS" == *","* ]]; then + FILTER_VAL=".*" + else + FILTER_VAL="${NETS}.${STAS}.${LOCS}.${CHAS}.*" + fi + + sed -i -E "s/\"WHITE\":\s*\[\s*\".*?\"\s*\]/\"WHITE\": [\"$FILTER_VAL\"]/" {{ config_path }} + args: + executable: /bin/bash + + - name: Read config.json after patch + slurp: + src: "{{ config_path }}" + register: config_after + + - name: "DISPLAY WHITE FILTER AFTER PATCH" + debug: + msg: + - "========================================" + - " WFCatalog Clean — config.json AFTER patch" + - "========================================" + - "{{ (config_after.content | b64decode | from_json).FILTERS }}" + - "========================================" + + # --------------------------------------------------------------- + # RUN COLLECTOR + # --------------------------------------------------------------- + + - name: Save file list to temporary text file + copy: + content: "{{ all_files | join('\n') }}" + dest: "/tmp/wfc_clean_{{ network }}_{{ station }}.txt" + + - name: Run WFCatalogCollector (clean) + shell: | + set -o pipefail + cd {{ collector_dir }} && \ + LIST_JSON=$(cat /tmp/wfc_clean_{{ network }}_{{ station }}.txt | jq -R . | jq -s -c .) && \ + {{ venv_python }} {{ collector_script }} \ + --list "$LIST_JSON" \ + --flags --csegs \ + --logfile {{ log_path }} \ + --stdout \ + --delete 2>&1 | tee {{ log_path }}.live + args: + executable: /bin/bash + async: 3600 + poll: 10 + register: wfcatalog_result + + - name: Read live log file after collector finishes + slurp: + src: "{{ log_path }}.live" + register: live_log + ignore_errors: true + + - name: "DISPLAY COLLECTOR STDOUT" + debug: + msg: + - "========================================" + - " WFCatalog Clean — Collector STDOUT" + - "========================================" + - "{{ wfcatalog_result.stdout | default('(no stdout)') }}" + - "========================================" + + - name: "DISPLAY LIVE LOG CONTENTS" + debug: + msg: + - "========================================" + - " WFCatalog Clean — Live Log ({{ log_path }}.live)" + - "========================================" + - "{{ (live_log.content | b64decode) if live_log.content is defined else '(live log not found)' }}" + - "========================================" + + - name: "DISPLAY COLLECTOR STDERR" + debug: + msg: + - "========================================" + - " WFCatalog Clean — Collector STDERR" + - "========================================" + - "{{ wfcatalog_result.stderr | default('(no stderr)') }}" + - "========================================" + + - name: "DISPLAY EXIT CODE" + debug: + msg: "Collector exited with rc={{ wfcatalog_result.rc | default('unknown') }}" + + # --------------------------------------------------------------- + # RESTORE CONFIG + JOB TRACKING + # --------------------------------------------------------------- + + - name: Restore original config.json + copy: + src: "{{ config_path }}.back" + dest: "{{ config_path }}" + remote_src: true + + - name: Ensure ~/.dmtri/jobs/{{ inventory_hostname }} directory exists + file: + path: "~/.dmtri/jobs/{{ inventory_hostname }}" + state: directory + mode: '0755' + + - name: "Load existing job list (if exists)" + slurp: + src: "~/.dmtri/jobs/{{ inventory_hostname }}/wfcatalog.json" + register: job_file + ignore_errors: true + + - name: "Parse job list or default to []" + set_fact: + job_list: >- + {{ (job_file.content | b64decode | from_json) if job_file.content is defined else [] }} + + - name: Define new job entry + set_fact: + new_job: { + "type": "wfcatalog_clean", + "job_id": "clean-{{ lookup('pipe', 'date +%s') }}", + "host": "{{ inventory_hostname }}", + "network": "{{ network }}", + "station": "{{ station }}", + "location": "{{ location }}", + "channel": "{{ channel }}", + "start": "{{ starttime }}", + "end": "{{ endtime }}", + "rc": "{{ wfcatalog_result.rc | default('unknown') }}", + "files_processed": "{{ all_files | length }}", + "created_at": "{{ lookup('pipe', 'date -u +%Y-%m-%dT%H:%M:%SZ') }}" + } + + - name: "Save updated job list" + copy: + content: "{{ (job_list + [new_job]) | to_nice_json }}" + dest: "~/.dmtri/jobs/{{ inventory_hostname }}/wfcatalog.json" + + - name: "DISPLAY FINAL SUMMARY" + debug: + msg: + - "========================================" + - " WFCatalog Clean — DONE" + - "========================================" + - " Files cleaned : {{ all_files | length }}" + - " Exit code : {{ wfcatalog_result.rc | default('unknown') }}" + - " Log file : {{ log_path }}" + - " Job ID : {{ new_job.job_id }}" + - "========================================" \ No newline at end of file diff --git a/src/dmtri/data/playbooks/refresh/availability_refresh.yml b/src/dmtri/data/playbooks/refresh/availability_refresh.yml new file mode 100644 index 0000000..582c346 --- /dev/null +++ b/src/dmtri/data/playbooks/refresh/availability_refresh.yml @@ -0,0 +1,68 @@ +- name: "Run ws-availability availability update (Webservice Command: Docker Restart)" + hosts: ws_availability + gather_facts: false + + tasks: + + - name: "Run availability update (Restarting Cacher Container)" + shell: "docker restart fdsnws-availability-cacher" + register: restart_result + + - name: "DISPLAY RESTART STATUS" + debug: + msg: + - "========================================" + - " Availability Refresh — Docker Status" + - "========================================" + - " Container : fdsnws-availability-cacher" + - " Status : {{ 'Restarted successfully' if restart_result.rc == 0 else 'FAILED' }}" + - " Exit Code : {{ restart_result.rc }}" + - "========================================" + + # --------------------------------------------------------------- + # JOB TRACKING + # --------------------------------------------------------------- + + - name: Ensure ~/.dmtri/jobs/{{ inventory_hostname }} directory exists + file: + path: "~/.dmtri/jobs/{{ inventory_hostname }}" + state: directory + mode: '0755' + + - name: "Load existing job list (if exists)" + slurp: + src: "~/.dmtri/jobs/{{ inventory_hostname }}/availability.json" + register: job_file + ignore_errors: true + + - name: "Parse job list or default to []" + set_fact: + job_list: >- + {{ (job_file.content | b64decode | from_json) if job_file.content is defined else [] }} + + - name: Define new job entry + set_fact: + new_job: { + "type": "availability_refresh", + "job_id": "restart-{{ lookup('pipe', 'date +%s') }}", + "host": "{{ inventory_hostname }}", + "mode": "full_restart", + "rc": "{{ restart_result.rc | default('unknown') }}", + "created_at": "{{ lookup('pipe', 'date -u +%Y-%m-%dT%H:%M:%SZ') }}" + } + + - name: "Save updated job list" + copy: + content: "{{ (job_list + [new_job]) | to_nice_json }}" + dest: "~/.dmtri/jobs/{{ inventory_hostname }}/availability.json" + + - name: "DISPLAY AVAILABILITY REFRESH RESULTS" + debug: + msg: + - "========================================" + - " Availability Refresh — Results" + - "========================================" + - " Status : {{ 'SUCCESS' if restart_result.rc == 0 else 'FAILED' }}" + - " Job ID : {{ new_job.job_id }}" + - " Message : The cacher container has been restarted and is now updating its internal database." + - "========================================" diff --git a/src/dmtri/data/playbooks/refresh/seedpsd_metadata_refresh.yml b/src/dmtri/data/playbooks/refresh/seedpsd_metadata_refresh.yml new file mode 100644 index 0000000..4a5a2ef --- /dev/null +++ b/src/dmtri/data/playbooks/refresh/seedpsd_metadata_refresh.yml @@ -0,0 +1,142 @@ +- name: Refresh SeedPSD metadata for a given network/station/time range + hosts: seedpsd_nodes + gather_facts: false + + tasks: + + - name: Ensure 'seedpsd_dir' is defined for this host + fail: + msg: "'seedpsd_dir' must be set in the inventory for {{ inventory_hostname }}" + when: seedpsd_dir is not defined + + - name: Ensure required variables are provided + fail: + msg: "You must pass 'network', 'station', 'starttime', and 'endtime' via -e" + when: network is not defined or station is not defined or starttime is not defined or endtime is not defined + + - name: Set defaults for optional variables + set_fact: + network: "{{ network | default('*') }}" + station: "{{ station | default('*') }}" + location: "{{ location | default('*') }}" + channel: "{{ channel | default('*') }}" + + - name: Set derived dates + set_fact: + start_date: "{{ starttime | regex_replace('T.*$', '') }}" + end_date: "{{ endtime | regex_replace('T.*$', '') }}" + network_list: "{{ [network] | flatten | join(',') }}" + station_list: "{{ [station] | flatten | join(',') }}" + channel_list: "{{ [channel] | flatten | join(',') }}" + + - name: Set log path + set_fact: + log_path: "{{ seedpsd_dir }}/log/SeedPSD-metadata-{{ network_list | regex_replace(',', '_') }}-{{ station_list | regex_replace(',', '_') }}-{{ start_date }}.log" + + - name: Ensure SeedPSD log directory exists + file: + path: "{{ seedpsd_dir }}/log" + state: directory + mode: '0755' + + - name: "[DEBUG] Show metadata refresh parameters" + debug: + msg: + - "========================================" + - " SeedPSD Metadata Refresh — Parameters" + - "========================================" + - " Host : {{ inventory_hostname }}" + - " Network : {{ network }}" + - " Station : {{ station }}" + - " Location : {{ location }}" + - " Channel : {{ channel }}" + - " Start date : {{ start_date }}" + - " End date : {{ end_date }}" + - " Log file : {{ log_path }}" + - "========================================" + - " PRO TIP: You can monitor progress live on the server with:" + - " tail -f {{ log_path }}.live" + - "========================================" + + + - name: Run seedpsd-cli metadata update + shell: | + set -o pipefail + cd {{ seedpsd_dir }} && \ + uv run seedpsd-cli metadata update \ + --network '{{ network_list }}' \ + --station '{{ station_list }}' \ + --location '{{ location }}' \ + --channel '{{ channel_list }}' \ + 2>&1 | tee {{ log_path }}.live + args: + executable: /bin/bash + async: 3600 + poll: 10 + register: metadata_result + + - name: Read live log file after metadata update finishes + slurp: + src: "{{ log_path }}.live" + register: live_log + ignore_errors: true + + - name: "DISPLAY METADATA REFRESH RESULTS" + debug: + msg: + - "========================================" + - " SeedPSD Metadata — Results" + - "========================================" + - " Status : {{ 'SUCCESS' if metadata_result.rc == 0 else 'FAILED' }}" + - " Log File : {{ log_path }}" + - "----------------------------------------" + - " Summary :" + - "{{ (live_log.content | b64decode | regex_findall('\\[INFO\\] (.*)', multiline=True) | length > 0) | ternary((live_log.content | b64decode | regex_findall('\\[INFO\\] (.*)', multiline=True)) | join('\n - '), 'No informative logs found') }}" + - "========================================" + when: true + + # --------------------------------------------------------------- + # JOB TRACKING + # --------------------------------------------------------------- + + - name: Ensure ~/.dmtri/jobs/{{ inventory_hostname }} directory exists + file: + path: "~/.dmtri/jobs/{{ inventory_hostname }}" + state: directory + mode: '0755' + + - name: "Load existing job list (if exists)" + slurp: + src: "~/.dmtri/jobs/{{ inventory_hostname }}/seedpsd-metadata.json" + register: job_file + ignore_errors: true + + - name: "Parse job list or default to []" + set_fact: + job_list: >- + {{ (job_file.content | b64decode | from_json) if job_file.content is defined else [] }} + + - name: Define new job entry + set_fact: + new_job: { + "type": "seedpsd_metadata_refresh", + "job_id": "metadata-{{ lookup('pipe', 'date +%s') }}", + "host": "{{ inventory_hostname }}", + "network": "{{ network }}", + "station": "{{ station }}", + "location": "{{ location }}", + "channel": "{{ channel }}", + "start": "{{ starttime }}", + "end": "{{ endtime }}", + "rc": "{{ metadata_result.rc | default('unknown') }}", + "created_at": "{{ lookup('pipe', 'date -u +%Y-%m-%dT%H:%M:%SZ') }}" + } + + - name: "Save updated job list" + copy: + content: "{{ (job_list + [new_job]) | to_nice_json }}" + dest: "~/.dmtri/jobs/{{ inventory_hostname }}/seedpsd-metadata.json" + + - name: "Job tracking summary" + debug: + msg: "Job {{ new_job.job_id }} recorded for {{ inventory_hostname }}" diff --git a/src/dmtri/data/playbooks/refresh/seedpsd_refresh.yml b/src/dmtri/data/playbooks/refresh/seedpsd_refresh.yml new file mode 100644 index 0000000..cb59110 --- /dev/null +++ b/src/dmtri/data/playbooks/refresh/seedpsd_refresh.yml @@ -0,0 +1,167 @@ +- name: Refresh SeedPSD data for a given network/station/time range + hosts: seedpsd_nodes + gather_facts: false + + vars: + # 'seedpsd_dir' should be defined in inventory + # archive_root is usually assumed to be 'iso_sds' relative to seedpsd_dir or absolute + + tasks: + - name: Ensure 'seedpsd_dir' is defined for this host + fail: + msg: "'seedpsd_dir' must be set in the inventory for {{ inventory_hostname }}" + when: seedpsd_dir is not defined + + - name: Ensure required variables are provided + fail: + msg: "You must pass 'network', 'station', 'starttime', and 'endtime' via -e" + when: network is not defined or station is not defined or starttime is not defined or endtime is not defined + + - name: Set variables + set_fact: + file_suffix: "{{ 'D' if 'data' in type | default('data') else 'M' }}" + network_list: "{{ [network] | flatten }}" + station_list: "{{ [station] | flatten }}" + channel_list: "{{ [channel | default('*')] | flatten }}" + start_year: "{{ starttime | regex_replace('^([0-9]{4}).*$', '\\1') | int }}" + end_year: "{{ endtime | regex_replace('^([0-9]{4}).*$', '\\1') | int }}" + + - name: Compute list of years to search + set_fact: + year_range: "{{ range(start_year | int, (end_year | int) + 1) | list }}" + + - name: "[Discovery] Generate list of -name patterns for DOY matching" + shell: | + start="{{ starttime | regex_replace('T.*$', '') }}" + end="{{ endtime | regex_replace('T.*$', '') }}" + tmp_list="" + + start_epoch=$(date -d "$start" +%s) + end_epoch=$(date -d "$end" +%s) + days_diff=$(( (end_epoch - start_epoch) / 86400 )) + + for i in $(seq 0 $days_diff); do + current_date=$(date -d "@$((start_epoch + i * 86400))" +%Y.%j) + year=$(echo "$current_date" | cut -d. -f1) + doy=$(echo "$current_date" | cut -d. -f2) + tmp_list="$tmp_list -name '*{{ file_suffix }}.${year}.${doy}' -o" + done + + # Remove trailing -o + echo "${tmp_list% -o}" + args: + executable: /bin/bash + register: name_patterns + changed_when: false + + - name: "[Discovery] Find SDS files across years" + shell: > + find iso_sds/{{ item_year }}/{{ net }}/{{ sta }}/{{ chan }}.{{ file_suffix }}/ -type f {{ name_patterns.stdout }} + loop: "{{ query('nested', year_range, network_list, station_list, channel_list) }}" + vars: + item_year: "{{ item[0] }}" + net: "{{ item[1] }}" + sta: "{{ item[2] }}" + chan: "{{ item[3] }}" + args: + executable: /bin/bash + register: find_results + changed_when: false + ignore_errors: true + + - name: "[Discovery] Collect found SDS file paths" + set_fact: + all_files: >- + {{ find_results.results + | map(attribute='stdout_lines') + | flatten + | select('string') + | list }} + + - name: "DISPLAY DISCOVERY SUMMARY" + debug: + msg: + - "========================================" + - " SeedPSD Discovery Summary" + - "========================================" + - " Files found : {{ all_files | length }}" + - " Years : {{ year_range | join(', ') }}" + - " Suffix : {{ file_suffix }}" + - "========================================" + when: true + + - name: "Check if files were found" + fail: + msg: "No SDS files found for the given criteria." + when: all_files | length == 0 + + - name: Write matched files to temporary list + copy: + content: "{{ all_files | join('\n') }}" + dest: "/tmp/seedpsd_refresh_{{ network }}_{{ station }}.txt" + + - name: Run seedpsd-cli data load with --list and --update + shell: | + set -o pipefail + cd {{ seedpsd_dir }} && \ + uv run seedpsd-cli data mseed \ + --update \ + --list "/tmp/seedpsd_refresh_{{ network }}_{{ station }}.txt" \ + 2>&1 | tee seedpsd_last_run.log + args: + executable: /bin/bash + register: seedpsd_result + + - name: "DISPLAY SEEDPSD DATA RESULTS" + debug: + msg: + - "========================================" + - " SeedPSD Data — Results" + - "========================================" + - " Status : {{ 'SUCCESS' if seedpsd_result.rc == 0 else 'FAILED' }}" + - " Files : {{ all_files | length }}" + - "----------------------------------------" + - " Summary :" + - "{{ (seedpsd_result.stdout | regex_findall('\\[INFO\\] (.*)', multiline=True) | length > 0) | ternary((seedpsd_result.stdout | regex_findall('\\[INFO\\] (.*)', multiline=True)) | join('\n - '), 'No informative logs found') }}" + - "========================================" + when: true + + # --------------------------------------------------------------- + # JOB TRACKING + # --------------------------------------------------------------- + + - name: Ensure ~/.dmtri/jobs/{{ inventory_hostname }} directory exists + file: + path: "~/.dmtri/jobs/{{ inventory_hostname }}" + state: directory + mode: '0755' + + - name: "Load existing job list" + slurp: + src: "~/.dmtri/jobs/{{ inventory_hostname }}/seedpsd-data.json" + register: job_file + ignore_errors: true + + - name: "Parse job list" + set_fact: + job_list: "{{ (job_file.content | b64decode | from_json) if job_file.content is defined else [] }}" + + - name: Define new job entry + set_fact: + new_job: { + "type": "seedpsd_refresh", + "job_id": "refresh-{{ lookup('pipe', 'date +%s') }}", + "host": "{{ inventory_hostname }}", + "network": "{{ network }}", + "station": "{{ station }}", + "start": "{{ starttime }}", + "end": "{{ endtime }}", + "files_processed": "{{ all_files | length }}", + "rc": "{{ seedpsd_result.rc }}", + "created_at": "{{ lookup('pipe', 'date -u +%Y-%m-%dT%H:%M:%SZ') }}" + } + + - name: "Save updated job list" + copy: + content: "{{ (job_list + [new_job]) | to_nice_json }}" + dest: "~/.dmtri/jobs/{{ inventory_hostname }}/seedpsd-data.json" diff --git a/src/playbooks/refresh/seedpsd_refresh_withtmp.yml b/src/dmtri/data/playbooks/refresh/seedpsd_refresh_withtmp.yml similarity index 90% rename from src/playbooks/refresh/seedpsd_refresh_withtmp.yml rename to src/dmtri/data/playbooks/refresh/seedpsd_refresh_withtmp.yml index 3335938..93a297c 100644 --- a/src/playbooks/refresh/seedpsd_refresh_withtmp.yml +++ b/src/dmtri/data/playbooks/refresh/seedpsd_refresh_withtmp.yml @@ -1,93 +1,93 @@ -- name: Find SDS files and run seedpsd-cli with accurate SDS path and DOY matching across years - hosts: seedpsd_nodes - gather_facts: false - vars: - description: "Finds SDS files ending in .D.YYYY.DDD across correct SDS folders, then runs seedpsd-cli with --list" - tasks: - - - name: Set variables - set_fact: - file_suffix: "{{ 'D' if 'data' in type else 'M' }}" - network: "{{ network | default('*') }}" - station: "{{ station | default('*') }}" - location: "{{ location | default('') }}" - channel: "{{ channel | default('*') }}" - # Note: These defaults are already set by argparse, so this is safe but redundant. - - name: Compute list of years to search - set_fact: - year_range: >- - {{ range( - (starttime | regex_replace('T.*$', '') | to_datetime('%Y-%m-%d')).year, - (endtime | regex_replace('T.*$', '') | to_datetime('%Y-%m-%d')).year + 1 - ) | list }} - - - name: Generate list of -name patterns for DOY matching across years (fixed logic) - shell: | - start="{{ starttime | regex_replace('T.*$', '') }}" - end="{{ endtime | regex_replace('T.*$', '') }}" - tmp_list="" - - start_epoch=$(date -d "$start" +%s) - end_epoch=$(date -d "$end" +%s) - days_diff=$(( (end_epoch - start_epoch) / 86400 )) - - for i in $(seq 0 $days_diff); do - current_date=$(date -d "@$((start_epoch + i * 86400))" +%Y.%j) - year=$(echo "$current_date" | cut -d. -f1) - doy=$(echo "$current_date" | cut -d. -f2) - tmp_list="$tmp_list -name '*{{ file_suffix }}.${year}.${doy}' -o" - done - - # Remove trailing -o - echo "${tmp_list% -o}" - args: - executable: /bin/bash - register: name_patterns - changed_when: false - - - name: Set final find arguments - set_fact: - find_args: "{{ name_patterns.stdout }}" - - - name: Run find in correct SDS folders per year - shell: > - find iso_sds/{{ item }}/{{ network }}/{{ station }}/{{ channel }}.{{ file_suffix }}/ -type f {{ find_args }} - loop: "{{ year_range }}" - args: - executable: /bin/bash - register: find_results - changed_when: false - ignore_errors: true - - - name: Collect found SDS file paths - set_fact: - all_files: >- - {{ find_results.results - | map(attribute='stdout_lines') - | flatten - | list }} - - - name: "Debug: Show matched file paths" - debug: - var: all_files - - - name: Write matched files to /tmp/matched_files.txt - copy: - content: "{{ all_files | join('\n') }}" - dest: /tmp/matched_files.txt - - - name: Show count of matched files - debug: - msg: "Total matched files: {{ all_files | length }}" - - - name: Run seedpsd-cli with --simulate and --list - shell: | - cd /home/qc/seedpsd && - uv run seedpsd-cli data mseed --simulate --list /tmp/matched_files.txt - args: - executable: /bin/bash - register: seedpsd_result - - - name: Show seedpsd-cli output - debug: - var: seedpsd_result.stdout_lines +- name: Find SDS files and run seedpsd-cli with accurate SDS path and DOY matching across years + hosts: seedpsd_nodes + gather_facts: false + vars: + description: "Finds SDS files ending in .D.YYYY.DDD across correct SDS folders, then runs seedpsd-cli with --list" + tasks: + + - name: Set variables + set_fact: + file_suffix: "{{ 'D' if 'data' in type else 'M' }}" + network: "{{ network | default('*') }}" + station: "{{ station | default('*') }}" + location: "{{ location | default('') }}" + channel: "{{ channel | default('*') }}" + # Note: These defaults are already set by argparse, so this is safe but redundant. + - name: Compute list of years to search + set_fact: + year_range: >- + {{ range( + (starttime | regex_replace('T.*$', '') | to_datetime('%Y-%m-%d')).year, + (endtime | regex_replace('T.*$', '') | to_datetime('%Y-%m-%d')).year + 1 + ) | list }} + + - name: Generate list of -name patterns for DOY matching across years (fixed logic) + shell: | + start="{{ starttime | regex_replace('T.*$', '') }}" + end="{{ endtime | regex_replace('T.*$', '') }}" + tmp_list="" + + start_epoch=$(date -d "$start" +%s) + end_epoch=$(date -d "$end" +%s) + days_diff=$(( (end_epoch - start_epoch) / 86400 )) + + for i in $(seq 0 $days_diff); do + current_date=$(date -d "@$((start_epoch + i * 86400))" +%Y.%j) + year=$(echo "$current_date" | cut -d. -f1) + doy=$(echo "$current_date" | cut -d. -f2) + tmp_list="$tmp_list -name '*{{ file_suffix }}.${year}.${doy}' -o" + done + + # Remove trailing -o + echo "${tmp_list% -o}" + args: + executable: /bin/bash + register: name_patterns + changed_when: false + + - name: Set final find arguments + set_fact: + find_args: "{{ name_patterns.stdout }}" + + - name: Run find in correct SDS folders per year + shell: > + find iso_sds/{{ item }}/{{ network }}/{{ station }}/{{ channel }}.{{ file_suffix }}/ -type f {{ find_args }} + loop: "{{ year_range }}" + args: + executable: /bin/bash + register: find_results + changed_when: false + ignore_errors: true + + - name: Collect found SDS file paths + set_fact: + all_files: >- + {{ find_results.results + | map(attribute='stdout_lines') + | flatten + | list }} + + - name: "DISPLAY MATCHED FILE PATHS" + debug: + var: all_files + + - name: Write matched files to /tmp/matched_files.txt + copy: + content: "{{ all_files | join('\n') }}" + dest: /tmp/matched_files.txt + + - name: "DISPLAY MATCHED FILE COUNT" + debug: + msg: "Total matched files: {{ all_files | length }}" + + - name: Run seedpsd-cli with --list + shell: | + cd /home/qc/seedpsd && + uv run seedpsd-cli data mseed --list /tmp/matched_files.txt + args: + executable: /bin/bash + register: seedpsd_result + + - name: "DISPLAY SEEDPSD OUTPUT" + debug: + var: seedpsd_result.stdout_lines diff --git a/src/dmtri/data/playbooks/refresh/wfcatalog_refresh.yml b/src/dmtri/data/playbooks/refresh/wfcatalog_refresh.yml new file mode 100644 index 0000000..7be4324 --- /dev/null +++ b/src/dmtri/data/playbooks/refresh/wfcatalog_refresh.yml @@ -0,0 +1,224 @@ +- name: Refresh WFCatalog metrics for a given network/station/time range + hosts: wf_catalogue + gather_facts: no + + vars: + venv_python: "{{ collector_dir }}/.env/bin/python3.11" + collector_script: "WFCatalogCollector.py" + config_path: "{{ collector_dir }}/config.json" + archive_root: "/home/sysop/iso_sds" + + tasks: + - name: Ensure required variables are provided + fail: + msg: "You must pass 'network', 'station', 'starttime', and 'endtime' via -e" + when: network is not defined or station is not defined or starttime is not defined or endtime is not defined + + - name: Set defaults for optional variables + set_fact: + network: "{{ network | default('*') }}" + station: "{{ station | default('*') }}" + location: "{{ location | default('*') }}" + channel: "{{ channel | default('*') }}" + file_suffix: 'D' + + - name: Set sanitized names for logging + set_fact: + net_str: "{{ (network if network is string else network | join('_')) | regex_replace('[^a-zA-Z0-9_*]', '') }}" + sta_str: "{{ (station if station is string else station | join('_')) | regex_replace('[^a-zA-Z0-9_*]', '') }}" + start_date: "{{ starttime | regex_replace('T.*$', '') }}" + end_date: "{{ endtime | regex_replace('T.*$', '') }}" + + - name: "[Discovery] Find files via targeted SDS path generation" + shell: + cmd: | + {{ venv_python }} -c " + import os, datetime, json, glob, fnmatch + + # Load config to get ARCHIVE_ROOT and STRUCTURE + with open('config.json') as f: cfg = json.load(f) + root = cfg.get('ARCHIVE_ROOT', '.') + struct = cfg.get('STRUCTURE', 'SDS') + + # Match patterns from Ansible + def get_list(env_var): + val = os.environ.get(env_var, '[]') + try: + data = json.loads(val) + return data if isinstance(data, list) else [data] + except: + return [val] if val else [] + + networks = get_list('DMTRI_NET') + stations = get_list('DMTRI_STA') + channels = get_list('DMTRI_CHAN') + + # Fix start/end parsing to handle possible timezone offset from isoformat() + start_str = '{{ starttime }}'.split('+')[0].split('Z')[0] + end_str = '{{ endtime }}'.split('+')[0].split('Z')[0] + start = datetime.datetime.strptime(start_str, '%Y-%m-%dT%H:%M:%S') + end = datetime.datetime.strptime(end_str, '%Y-%m-%dT%H:%M:%S') + + found = [] + curr = start + while curr <= (end + datetime.timedelta(seconds=1)): + year, doy = curr.strftime('%Y'), curr.strftime('%j') + + for net_p in networks: + for sta_p in stations: + for chan_p in channels: + if struct == 'SDS': + # root/year/net/sta/chan.type/net.sta.loc.chan.type.year.doy + pattern = os.path.join(root, year, net_p, sta_p, f'{chan_p}.*', f'{net_p}.{sta_p}.*.{chan_p}.*.{year}.{doy}') + found.extend(glob.glob(pattern)) + elif struct == 'ODC': + # root/year/doy/sta.chan.net.year.doy + pattern = os.path.join(root, year, doy, f'{sta_p}.{chan_p}.{net_p}.{year}.{doy}') + found.extend(glob.glob(pattern)) + + curr += datetime.timedelta(days=1) + + # Remove duplicates and filter by config filters + found = sorted(list(set(found))) + white = cfg.get('FILTERS', {}).get('WHITE', ['*']) + black = cfg.get('FILTERS', {}).get('BLACK', []) + + final = [] + for f in found: + base = os.path.basename(f) + if any(fnmatch.fnmatch(base, w) for w in white): + if not any(fnmatch.fnmatch(base, b) for b in black): + final.append(f) + + print(json.dumps(final)) + " + chdir: "{{ collector_dir }}" + args: + executable: /bin/bash + environment: + DMTRI_NET: "{{ network | to_json }}" + DMTRI_STA: "{{ station | to_json }}" + DMTRI_CHAN: "{{ channel | to_json }}" + register: discovery_result + + - name: "[Discovery] Set file list" + set_fact: + all_files: "{{ discovery_result.stdout | from_json }}" + + - name: "[Discovery] Check if files were found" + fail: + msg: "No files found for {{ network }}.{{ station }} between {{ starttime }} and {{ endtime }}." + when: all_files | length == 0 + + - name: Create temporary workspace for isolation + tempfile: + state: directory + suffix: wfcatalog + register: temp_workspace + + - name: Prepare isolated workspace + copy: + src: "{{ collector_dir }}/{{ item }}" + dest: "{{ temp_workspace.path }}/{{ item }}" + remote_src: true + loop: + - "{{ collector_script }}" + - "config.json" + + - name: Process files in batches + shell: + cmd: | + set -o pipefail + LOG_FILE="WFCatalog-refresh-{{ net_str }}-{{ sta_str }}-{{ start_date }}-{{ end_date }}-batch-{{ ansible_loop.index }}.log" + {{ venv_python }} {{ collector_script }} \ + --list '{{ item | to_json }}' \ + --update --force --csegs --flags \ + --stdout 2>&1 | tee "$LOG_FILE" + chdir: "{{ temp_workspace.path }}" + args: + executable: /bin/bash + environment: + PYTHONPATH: "{{ collector_dir }}" + PYTHONUNBUFFERED: "1" + register: batch_results + loop: "{{ all_files | batch(100) | list }}" + loop_control: + extended: yes + label: "Batch {{ ansible_loop.index }} ({{ item | length }} files)" + + + - name: Ensure permanent log directory exists + file: + path: "{{ collector_dir }}/log" + state: directory + mode: '0755' + + - name: Persist batch log + shell: "cp {{ temp_workspace.path }}/*.log {{ collector_dir }}/log/" + args: + executable: /bin/bash + ignore_errors: true + + - name: Remove temporary workspace + file: + path: "{{ temp_workspace.path }}" + state: absent + when: temp_workspace.path is defined + + - name: Ensure ~/.dmtri/jobs/{{ inventory_hostname }} directory exists + file: + path: "~/.dmtri/jobs/{{ inventory_hostname }}" + state: directory + mode: '0755' + + - name: Load existing job list + slurp: + src: "~/.dmtri/jobs/{{ inventory_hostname }}/wfcatalog.json" + register: job_file + ignore_errors: true + + - name: Parse job list + set_fact: + job_list: "{{ (job_file.content | b64decode | from_json) if job_file.content is defined else [] }}" + + - name: Define new job entry + set_fact: + new_job: { + "type": "wfcatalog_refresh", + "job_id": "refresh-{{ lookup('pipe', 'date +%s') }}", + "host": "{{ inventory_hostname }}", + "network": "{{ network }}", + "station": "{{ station }}", + "start": "{{ starttime }}", + "end": "{{ endtime }}", + "files_processed": "{{ all_files | length }}", + "batches": "{{ batch_results.results | length }}", + "created_at": "{{ lookup('pipe', 'date -u +%Y-%m-%dT%H:%M:%SZ') }}" + } + + - name: Append new job and save + copy: + content: "{{ (job_list + [new_job]) | to_nice_json }}" + dest: "~/.dmtri/jobs/{{ inventory_hostname }}/wfcatalog.json" + + - name: Read last batch log for summary + slurp: + src: "{{ collector_dir }}/log/WFCatalog-refresh-{{ net_str }}-{{ sta_str }}-{{ start_date }}-{{ end_date }}-batch-1.log" + register: last_batch_log + ignore_errors: true + + - name: "DISPLAY WFCATALOG REFRESH RESULTS" + debug: + msg: + - "========================================" + - " WFCatalog Refresh — Results" + - "========================================" + - " Status : {{ 'SUCCESS' if batch_results.results | map(attribute='rc') | sum == 0 else 'PARTIAL FAILURE' }}" + - " Files : {{ all_files | length }}" + - " Batches : {{ batch_results.results | length }}" + - " Log Dir : {{ collector_dir }}/log" + - "----------------------------------------" + - " Summary :" + - "{{ (last_batch_log.content | b64decode | regex_findall('Total records: (.*)', multiline=True) | length > 0) | ternary((last_batch_log.content | b64decode | regex_findall('Total records: (.*)', multiline=True)) | first, 'Summary info not found in log') }}" + - " (Job ID: {{ new_job.job_id }})" + - "========================================" \ No newline at end of file diff --git a/src/playbooks/seedpsd_simulate.yml b/src/dmtri/data/playbooks/seedpsd_simulate.yml similarity index 100% rename from src/playbooks/seedpsd_simulate.yml rename to src/dmtri/data/playbooks/seedpsd_simulate.yml diff --git a/src/dmtri/data/playbooks/track.yml b/src/dmtri/data/playbooks/track.yml new file mode 100644 index 0000000..8297fe9 --- /dev/null +++ b/src/dmtri/data/playbooks/track.yml @@ -0,0 +1,56 @@ +- name: Track latest job from each JSON file per host + hosts: all + gather_facts: false + + vars: + latest_jobs: [] + + tasks: + - name: Gather remote environment variables + setup: + filter: ansible_env + + - name: Find job tracking files in ~/.dmtri/jobs/{{ inventory_hostname }} + find: + paths: "{{ ansible_env.HOME }}/.dmtri/jobs/{{ inventory_hostname }}" + patterns: "*.json" + file_type: file + register: job_files + ignore_errors: true + + - name: Fail if no job files were found + fail: + msg: "No job tracking files found for {{ inventory_hostname }}" + when: job_files.files | length == 0 + + - name: Load and decode each job file + slurp: + src: "{{ item.path }}" + loop: "{{ job_files.files }}" + register: loaded_jobs + loop_control: + label: "{{ item.path }}" + + - name: Try parsing and selecting latest job from each file + set_fact: + latest_jobs: "{{ latest_jobs + [ (item.content | b64decode | from_json | sort(attribute='created_at'))[-1] ] }}" + failed_when: false + loop: "{{ loaded_jobs.results }}" + loop_control: + label: "{{ item.source }}" + when: item.content is defined + + - name: Print latest job from each file + debug: + msg: >- + {{ '%-18s' | format(item.host | default(inventory_hostname)) }} + {{ '%-28s' | format(item.job_id) }} + {{ '%-22s' | format((item.type ~ ' (' ~ item.subtype ~ ')') if item.subtype is defined else item.type) }} + {{ '%-10s' | format(item.network | default('—')) }} + {{ '%-10s' | format(item.station | default('—')) }} + {{ '%-20s' | format(item.start | default('—')) }} + {{ '%-20s' | format(item.end | default('—')) }} + {{ '(MANUAL)' if item.job_id is search('^manual-') else '(AUTO)' }} ✅ From file + loop: "{{ latest_jobs }}" + loop_control: + label: "{{ item.job_id }}" diff --git a/src/dmtri/doctor.py b/src/dmtri/doctor.py index 9e2ff72..3bfdb46 100644 --- a/src/dmtri/doctor.py +++ b/src/dmtri/doctor.py @@ -1,52 +1,54 @@ -import logging -import ansible_runner -from dmtri.paths import INVENTORY_FILE - -logger = logging.getLogger(__name__) - -def run_diagnostics(verbose=False, retries=1): - logger.info(f"Running connectivity check using inventory: {INVENTORY_FILE}") - - ok_hosts = set() - unreachable_hosts = set() - - for attempt in range(retries): - logger.info(f"Attempt {attempt + 1} of {retries}") - - r = ansible_runner.run( - private_data_dir=".", - inventory=str(INVENTORY_FILE.resolve()), - module="ping", - host_pattern="all", - quiet=not verbose, - envvars={ - "ANSIBLE_CACHE_PLUGIN": "memory" # Disable caching to avoid plugin warnings - } - ) - - for event in r.events: - if not isinstance(event, dict): - continue - data = event.get("event_data", {}) - host = data.get("host") - if not host: - continue - if event["event"] == "runner_on_ok": - ok_hosts.add(host) - elif event["event"] == "runner_on_unreachable": - unreachable_hosts.add(host) - - # Break early if all hosts responded - if ok_hosts and not unreachable_hosts: - break - - logger.info("\nResults:") - - if not ok_hosts and not unreachable_hosts: - logger.warning("No responses received. Are you connected to the VPN?") - return - - for host in sorted(ok_hosts): - logger.info(f"[OK] {host}") - for host in sorted(unreachable_hosts): - logger.error(f"[UNREACHABLE] {host}") +import logging +import ansible_runner +from dmtri.paths import INVENTORY_FILE + +logger = logging.getLogger(__name__) + +def run_diagnostics(verbose=False, retries=1, inventory=None): + if inventory is None: + inventory = INVENTORY_FILE + logger.info(f"Running connectivity check using inventory: {inventory}") + + ok_hosts = set() + unreachable_hosts = set() + + for attempt in range(retries): + logger.info(f"Attempt {attempt + 1} of {retries}") + + r = ansible_runner.run( + private_data_dir=".", + inventory=str(inventory.resolve()), + module="ping", + host_pattern="all", + quiet=not verbose, + envvars={ + "ANSIBLE_CACHE_PLUGIN": "memory" # Disable caching to avoid plugin warnings + } + ) + + for event in r.events: + if not isinstance(event, dict): + continue + data = event.get("event_data", {}) + host = data.get("host") + if not host: + continue + if event["event"] == "runner_on_ok": + ok_hosts.add(host) + elif event["event"] == "runner_on_unreachable": + unreachable_hosts.add(host) + + # Break early if all hosts responded + if ok_hosts and not unreachable_hosts: + break + + logger.info("\nResults:") + + if not ok_hosts and not unreachable_hosts: + logger.warning("No responses received. Are you connected to the VPN?") + return + + for host in sorted(ok_hosts): + logger.info(f"[OK] {host}") + for host in sorted(unreachable_hosts): + logger.error(f"[UNREACHABLE] {host}") diff --git a/src/dmtri/init.py b/src/dmtri/init.py new file mode 100644 index 0000000..e9b2646 --- /dev/null +++ b/src/dmtri/init.py @@ -0,0 +1,52 @@ +import shutil +import logging +from pathlib import Path +import appdirs + +from dmtri.paths import DATA_DIR + +logger = logging.getLogger(__name__) + +USER_CONFIG_DIR = Path(appdirs.user_config_dir("dmtri")) + + +def run_init(force=False): + """ + Copy bundled playbooks and inventory into the user config directory + so the user can customize them. + """ + if USER_CONFIG_DIR.exists() and not force: + logger.info(f"Config directory already exists: {USER_CONFIG_DIR}") + logger.info("Use --force to overwrite existing files.") + logger.info(f"\nYou can edit the file with: nano {USER_CONFIG_DIR}/hosts.ini") + return + + USER_CONFIG_DIR.mkdir(parents=True, exist_ok=True) + + # Copy inventory from example + src_inventory = DATA_DIR / "inventory" / "hosts.ini.example" + dst_inventory = USER_CONFIG_DIR / "hosts.ini" + if not dst_inventory.exists() or force: + shutil.copy2(src_inventory, dst_inventory) + logger.info(f" [created] {dst_inventory}") + logger.info(" ⚠️ Edit hosts.ini with your real host details before running any commands!") + else: + logger.info(f" [skipped] {dst_inventory} (already exists)") + + + # Copy playbooks tree + src_playbooks = DATA_DIR / "playbooks" + dst_playbooks = USER_CONFIG_DIR / "playbooks" + if dst_playbooks.exists() and force: + shutil.rmtree(dst_playbooks) + if not dst_playbooks.exists(): + shutil.copytree(src_playbooks, dst_playbooks) + logger.info(f" [created] {dst_playbooks}/") + else: + logger.info(f" [skipped] {dst_playbooks}/ (already exists, use --force to overwrite)") + + logger.info(f"\nDone! Edit your configuration in: {USER_CONFIG_DIR}") + logger.info(" - Inventory : " + str(USER_CONFIG_DIR / "hosts.ini")) + logger.info(" - Playbooks : " + str(USER_CONFIG_DIR / "playbooks/")) + logger.info(f"\nQuick edit command: nano {USER_CONFIG_DIR}/hosts.ini") + logger.info("\ndmtri will automatically use your local config on next run.") diff --git a/src/dmtri/paths.py b/src/dmtri/paths.py index 87d7e3b..4c15b8b 100644 --- a/src/dmtri/paths.py +++ b/src/dmtri/paths.py @@ -1,30 +1,94 @@ from pathlib import Path +import appdirs +import os MODULE_DIR = Path(__file__).resolve().parent -SRC_DIR = MODULE_DIR.parent +DATA_DIR = MODULE_DIR / "data" -# Directories -PLAYBOOKS_DIR = SRC_DIR / "playbooks" -INVENTORY_DIR = SRC_DIR / "inventory" +# Bundled directories (always available) +PLAYBOOKS_DIR = DATA_DIR / "playbooks" +INVENTORY_DIR = DATA_DIR / "inventory" -# Inventory file +# User config directory (populated by `dmtri init`) +USER_CONFIG_DIR = Path(appdirs.user_config_dir("dmtri")) +USER_PLAYBOOKS_DIR = USER_CONFIG_DIR / "playbooks" + +# Ansible config files (always bundled) +ANSIBLE_QUIET_CFG = DATA_DIR / "ansible_quiet.cfg" +ANSIBLE_TRACK_CFG = DATA_DIR / "ansible_track.cfg" +ANSIBLE_VERBOSE_CFG = DATA_DIR / "ansible_verbose.cfg" + +# Bundled inventory example (committed to git, safe to ship) +INVENTORY_EXAMPLE = INVENTORY_DIR / "hosts.ini.example" +# Local inventory — git-ignored, never bundled in the package, used for repo dev INVENTORY_FILE = INVENTORY_DIR / "hosts.ini" -# Specific playbooks -PLAYBOOK_TRACK = PLAYBOOKS_DIR / "track.yml" -PLAYBOOK_LIST_SDS = PLAYBOOKS_DIR / "list_sds_files.yml" - -# Refresh Data playbooks -PLAYBOOK_SEEDPSD_REFRESH = PLAYBOOKS_DIR / "refresh" / "seedpsd_refresh.yml" -PLAYBOOK_WFCATALOG_REFRESH = PLAYBOOKS_DIR / "refresh" / "wfcatalog_refresh.yml" -PLAYBOOK_AVAILABILITY_REFRESH = PLAYBOOKS_DIR / "refresh" / "availability_refresh.yml" -# Refresh metadata playbooks -PLAYBOOK_METADATA_REFRESH = PLAYBOOKS_DIR / "refresh" / "seedpsd_metadata_refresh.yml" - -# Clean playbooks -PLAYBOOK_SEEDPSD_CLEAN = PLAYBOOKS_DIR / "clean" /"seedpsd_clean.yml" -PLAYBOOK_WFCATALOG_CLEAN = PLAYBOOKS_DIR / "clean" / "wfcatalog_clean.yml" -PLAYBOOK_AVAILABILITY_CLEAN = PLAYBOOKS_DIR / "clean" / "availability_clean.yml" + +def get_inventory_path(override=None): + """ + Determine the inventory path using the following priority: + 1. CLI flag (--inventory) + 2. Environment variable (DMTRI_INVENTORY) + 3. User config directory (~/.config/dmtri/hosts.ini) + + Raises SystemExit with a helpful message if no inventory is found. + Run `dmtri init` to create your local hosts.ini from the example. + """ + import sys + if override: + p = Path(override).resolve() + if not p.exists(): + print(f"Error: inventory file not found: {p}", file=sys.stderr) + sys.exit(1) + return p + + env_val = os.environ.get("DMTRI_INVENTORY") + if env_val: + p = Path(env_val).resolve() + if not p.exists(): + print(f"Error: DMTRI_INVENTORY points to a missing file: {p}", file=sys.stderr) + sys.exit(1) + return p + + user_path = USER_CONFIG_DIR / "hosts.ini" + if user_path.exists(): + return user_path + + # Local dev fallback: repo clone with a git-ignored hosts.ini + if INVENTORY_FILE.exists(): + return INVENTORY_FILE + + print( + "Error: No inventory file found.\n" + "Run \"dmtri init\" to create your local hosts.ini from the bundled example,\n" + f"then edit it at: {USER_CONFIG_DIR / 'hosts.ini'}", + file=sys.stderr, + ) + sys.exit(1) + + +def _resolve_playbook(relative_path: str) -> Path: + """ + Return user playbook if it exists, otherwise fall back to bundled version. + relative_path is relative to the playbooks root, e.g. 'refresh/seedpsd_refresh.yml' + """ + user_pb = USER_PLAYBOOKS_DIR / relative_path + if user_pb.exists(): + return user_pb + return PLAYBOOKS_DIR / relative_path + + +# Playbooks — resolved dynamically so user overrides are respected +PLAYBOOK_TRACK = _resolve_playbook("track.yml") + +PLAYBOOK_SEEDPSD_REFRESH = _resolve_playbook("refresh/seedpsd_refresh.yml") +PLAYBOOK_WFCATALOG_REFRESH = _resolve_playbook("refresh/wfcatalog_refresh.yml") +PLAYBOOK_AVAILABILITY_REFRESH = _resolve_playbook("refresh/availability_refresh.yml") +PLAYBOOK_METADATA_REFRESH = _resolve_playbook("refresh/seedpsd_metadata_refresh.yml") + +PLAYBOOK_SEEDPSD_CLEAN = _resolve_playbook("clean/seedpsd_clean.yml") +PLAYBOOK_WFCATALOG_CLEAN = _resolve_playbook("clean/wfcatalog_clean.yml") +PLAYBOOK_AVAILABILITY_CLEAN = _resolve_playbook("clean/availability_clean.yml") # Group playbooks by command COMMAND_PLAYBOOKS = { diff --git a/src/dmtri/utils.py b/src/dmtri/utils.py index a667c35..b7ddaa3 100644 --- a/src/dmtri/utils.py +++ b/src/dmtri/utils.py @@ -4,12 +4,22 @@ from pathlib import Path import tomllib +from importlib.metadata import version, PackageNotFoundError + def get_version(): - root = Path(__file__).resolve().parents[2] - pyproject_path = root / "pyproject.toml" - with open(pyproject_path, "rb") as f: - data = tomllib.load(f) - return data["project"]["version"] + try: + # Try to get version from installed package metadata + return version("eida-dmtri") + except PackageNotFoundError: + # Fallback for local development where the package might not be installed + try: + root = Path(__file__).resolve().parents[2] + pyproject_path = root / "pyproject.toml" + with open(pyproject_path, "rb") as f: + data = tomllib.load(f) + return data.get("project", {}).get("version", "unknown") + except Exception: + return "unknown" def maybe_warn_shell_globbing(args): cwd_files = set(os.listdir(".")) @@ -37,8 +47,8 @@ def validate_datetime_range(start_dt, end_dt): print("Error: --endtime cannot be in the future.", file=sys.stderr) sys.exit(1) - if end_dt <= start_dt: - print("Error: --endtime must be after --starttime.", file=sys.stderr) + if end_dt < start_dt: + print("Error: --endtime must be on or after --starttime.", file=sys.stderr) sys.exit(1) duration_days = (end_dt - start_dt).days @@ -48,22 +58,44 @@ def validate_datetime_range(start_dt, end_dt): if proceed != "y": print("Aborted by user.") sys.exit(0) +def parse_flexible_date(value: str) -> datetime: + """Parses ISO 8601, partial date, or relative expressions like 'yesterday', '-1d', etc.""" + from datetime import timedelta + import re -def parse_datetime_args(args): - try: - start_dt = datetime.fromisoformat(args.starttime) - end_dt = datetime.fromisoformat(args.endtime) + value = value.strip().lower() + now = datetime.now(timezone.utc) - if start_dt.tzinfo is None: - start_dt = start_dt.replace(tzinfo=timezone.utc) - if end_dt.tzinfo is None: - end_dt = end_dt.replace(tzinfo=timezone.utc) + if value == "now": + return now + if value == "today": + return now.replace(hour=0, minute=0, second=0, microsecond=0) + if value == "yesterday": + return (now - timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0) + + # Relative offsets: +1d, -2h, +30m + match = re.fullmatch(r"([+-])(\d+)([dhms])", value) + if match: + sign, number, unit = match.groups() + number = int(number) + delta = {"d": timedelta(days=number), "h": timedelta(hours=number), + "m": timedelta(minutes=number), "s": timedelta(seconds=number)}[unit] + return now + delta if sign == "+" else now - delta + + # Try parsing ISO 8601 or partial formats + formats = ("%Y-%m-%d", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%dT%H:%M") + for fmt in formats: + try: + dt = datetime.strptime(value, fmt) + return dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt + except ValueError: + continue + + raise ValueError(f"Invalid date format: '{value}'") - return start_dt, end_dt +def parse_datetime_args(args): + return args.starttime, args.endtime - except ValueError: - print("Error: Invalid date format. Use ISO 8601 (e.g. 2025-01-01T00:00:00)", file=sys.stderr) - sys.exit(1) def validate_types(args): types = [t.strip() for t in args.type.split(",") if t.strip() in ("data", "metadata")] diff --git a/src/playbooks/clean/wfcatalog_clean.yml b/src/playbooks/clean/wfcatalog_clean.yml deleted file mode 100644 index 6552628..0000000 --- a/src/playbooks/clean/wfcatalog_clean.yml +++ /dev/null @@ -1,107 +0,0 @@ -- name: Run WFCatalog Clean - hosts: wf_catalogue - gather_facts: false - - vars: - venv_python: ".env/bin/python3.11" - collector_script: "WFCatalogCollector.py" - config_path: "{{ collector_dir }}/config.json" - - tasks: - - - name: Ensure required variables are provided - fail: - msg: "You must pass 'network', 'station', 'starttime', and 'endtime' via -e" - when: network is not defined or station is not defined or starttime is not defined or endtime is not defined - - - name: Compute basic derived values - set_fact: - start_date: "{{ starttime | regex_replace('T.*$', '') }}" - end_date: "{{ endtime | regex_replace('T.*$', '') }}" - year: "{{ starttime[0:4] }}" - - - name: Compute range and paths - set_fact: - range_days: >- - {{ - ( - (endtime | to_datetime('%Y-%m-%dT%H:%M:%S')) - - (starttime | to_datetime('%Y-%m-%dT%H:%M:%S')) - ).days + 1 - }} - log_path: "{{ collector_dir }}/log/WFCatalog-{{ network }}-{{ station }}-{{ start_date }}-{{ end_date }}.log" - custom_archive_root: "/home/sysop/iso_sds/{{ year }}/{{ network }}/{{ station }}" - - - name: Check if backup exists - stat: - path: "{{ config_path }}.back" - register: config_backup - - - name: Backup original config.json - copy: - src: "{{ config_path }}" - dest: "{{ config_path }}.back" - remote_src: true - when: not config_backup.stat.exists - - - name: Update ARCHIVE_ROOT in config.json - replace: - path: "{{ config_path }}" - regexp: '"ARCHIVE_ROOT":\s*".*?"' - replace: '"ARCHIVE_ROOT": "{{ custom_archive_root }}"' - - - name: Run WFCatalogCollector asynchronously - shell: | - cd {{ collector_dir }} && \ - {{ venv_python }} {{ collector_script }} \ - --flags --csegs \ - --logfile {{ log_path }} \ - --date {{ start_date }} \ - --range {{ range_days }} \ - --delete --stdout - args: - executable: /bin/bash - async: 3600 - poll: 0 - register: wfcatalog_async - - - name: "Track job: append to {{ collector_dir }}/.dmtri_jobs.json" - block: - - name: Load existing job list - slurp: - src: "{{ collector_dir }}/.dmtri_jobs.json" - register: job_file - ignore_errors: true - - - name: Parse or start new job list - set_fact: - job_list: >- - {{ - (job_file.content | b64decode | from_json) - if job_file is defined and job_file.content is defined - else [] - }} - - - name: Create job record - set_fact: - new_job: { - "type": "wfcatalog", - "job_id": "{{ wfcatalog_async.ansible_job_id }}", - "host": "{{ inventory_hostname }}", - "network": "{{ network }}", - "station": "{{ station }}", - "start": "{{ starttime }}", - "end": "{{ endtime }}", - "created_at": "{{ lookup('pipe', 'date -u +%Y-%m-%dT%H:%M:%SZ') }}" - } - - - name: Save updated job list - copy: - content: "{{ (job_list + [new_job]) | to_nice_json }}" - dest: "{{ collector_dir }}/.dmtri_jobs.json" - - - name: Restore original config.json - copy: - src: "{{ config_path }}.back" - dest: "{{ config_path }}" - remote_src: true \ No newline at end of file diff --git a/src/playbooks/refresh/availability_refresh.yml b/src/playbooks/refresh/availability_refresh.yml deleted file mode 100644 index bf6f5bb..0000000 --- a/src/playbooks/refresh/availability_refresh.yml +++ /dev/null @@ -1,82 +0,0 @@ -- name: "Run ws-availability availability update" - hosts: ws_availability - gather_facts: false - - vars: - mongo_script: main.js - - tasks: - - - name: "Ensure required vars are provided" - fail: - msg: "You must pass 'network', 'station', 'starttime', and 'endtime' via -e" - when: network is not defined or station is not defined or starttime is not defined or endtime is not defined - - - name: "Convert starttime/endtime to date-only format" - set_fact: - start_date: "{{ starttime | regex_replace('T.*$', '') }}" - end_date: "{{ endtime | regex_replace('T.*$', '') }}" - - - name: "Preview full mongosh command for verification" - debug: - msg: > - cd {{ availability_dir }} && - mongosh -u {{ mongo_user }} -p {{ mongo_pass }} - --authenticationDatabase {{ mongo_authdb }} - --eval "networks='{{ network }}'; stations='{{ station }}'; start='{{ start_date }}'; end='{{ end_date }}'" {{ mongo_script }} - - - name: "Run availability script via mongosh" - shell: > - cd {{ availability_dir }} && - mongosh -u {{ mongo_user }} -p {{ mongo_pass }} - --authenticationDatabase {{ mongo_authdb }} - --eval "networks='{{ network }}'; stations='{{ station }}'; start='{{ start_date }}'; end='{{ end_date }}'" {{ mongo_script }} - args: - executable: /bin/bash - register: availability_result - - - name: "Show mongosh output" - debug: - var: availability_result.stdout_lines - - - name: "Track job: keep only latest 'availability' job and preserve others" - block: - - - name: Ensure ~/.dmtri/jobs/{{ inventory_hostname }} directory exists - file: - path: "~/.dmtri/jobs/{{ inventory_hostname }}" - state: directory - mode: '0755' - - - name: "Load existing job list (if exists)" - slurp: - src: "~/.dmtri/jobs/{{ inventory_hostname }}/availability.json" - register: job_file - ignore_errors: true - - - name: "Parse job list or default to []" - set_fact: - job_list: >- - {{ (job_file.content | b64decode | from_json) if job_file.content is defined else [] }} - - - name: "Define new job entry" - set_fact: - new_job: { - "type": "availability", - "job_id": "manual-{{ lookup('pipe', 'date +%s') }}", - "host": "{{ inventory_hostname }}", - "network": "{{ network }}", - "station": "{{ station }}", - "start": "{{ starttime }}", - "end": "{{ endtime }}", - "created_at": "{{ lookup('pipe', 'date -u +%Y-%m-%dT%H:%M:%SZ') }}" - } - - - name: "Remove any previous job of the same type" - set_fact: - filtered_jobs: "{{ job_list | rejectattr('type', 'equalto', new_job.type) | list }}" - - - name: "Save updated job list (one job per type)" - copy: - content: "{{ (filtered_jobs + [new_job]) | to_nice_json }}" - dest: "~/.dmtri/jobs/{{ inventory_hostname }}/availability.json" diff --git a/src/playbooks/refresh/seedpsd_metadata_refresh.yml b/src/playbooks/refresh/seedpsd_metadata_refresh.yml deleted file mode 100644 index c6756f3..0000000 --- a/src/playbooks/refresh/seedpsd_metadata_refresh.yml +++ /dev/null @@ -1,79 +0,0 @@ -- name: Run seedpsd-cli metadata update in simulate mode - hosts: seedpsd_nodes - gather_facts: false - - vars: - description: "Run simulated metadata update using seedpsd-cli" - - tasks: - - - name: Collect remote environment variables - setup: - filter: ansible_env - - - name: Ensure required variables are provided - fail: - msg: "You must pass 'network', 'station', 'starttime', and 'endtime' via -e" - when: network is not defined or station is not defined or starttime is not defined or endtime is not defined - - - name: Ensure 'seedpsd_dir' is defined for this host - fail: - msg: "'seedpsd_dir' must be set in the inventory for {{ inventory_hostname }}" - when: seedpsd_dir is not defined - - - name: Compute derived values - set_fact: - start_date: "{{ starttime | regex_replace('T.*$', '') }}" - end_date: "{{ endtime | regex_replace('T.*$', '') }}" - year: "{{ starttime[0:4] }}" - subtype: "metadata" - - - name: Run seedpsd-cli metadata update in simulate mode - shell: > - cd {{ seedpsd_dir }} && - uv run seedpsd-cli metadata update - --network {{ network }} - --station {{ station }} - --location '{{ location | default('*') }}' - --channel '{{ channel | default('*') }}' - --simulate - args: - executable: /bin/bash - register: metadata_result - - - name: Show metadata update output - debug: - var: metadata_result.stdout_lines - - - name: Set remote job directory path - set_fact: - job_dir: "{{ ansible_env.HOME }}/.dmtri/jobs/{{ inventory_hostname }}" - - - name: Set remote job file path - set_fact: - job_path: "{{ job_dir }}/seedpsd-{{ subtype }}.json" - - - name: Ensure job directory exists - file: - path: "{{ job_dir }}" - state: directory - mode: '0755' - - - name: Define new job entry - set_fact: - new_job: { - "type": "seedpsd", - "subtype": "{{ subtype }}", - "job_id": "manual-{{ lookup('pipe', 'date +%s') }}", - "host": "{{ inventory_hostname }}", - "network": "{{ network }}", - "station": "{{ station }}", - "start": "{{ starttime }}", - "end": "{{ endtime }}", - "created_at": "{{ lookup('pipe', 'date -u +%Y-%m-%dT%H:%M:%SZ') }}" - } - - - name: Save latest metadata job (overwrite) - copy: - content: "{{ [new_job] | to_nice_json }}" - dest: "{{ job_path }}" diff --git a/src/playbooks/refresh/seedpsd_refresh.yml b/src/playbooks/refresh/seedpsd_refresh.yml deleted file mode 100644 index f85f65c..0000000 --- a/src/playbooks/refresh/seedpsd_refresh.yml +++ /dev/null @@ -1,143 +0,0 @@ -- name: Find SDS files and run seedpsd-cli with accurate SDS path and DOY matching across years - hosts: seedpsd_nodes - gather_facts: false - - tasks: - - - name: Ensure 'seedpsd_dir' is defined for this host - fail: - msg: "'seedpsd_dir' must be set in the inventory for {{ inventory_hostname }}" - when: seedpsd_dir is not defined - - - name: Set variables - set_fact: - file_suffix: 'D' - network: "{{ network | default('*') }}" - station: "{{ station | default('*') }}" - location: "{{ location | default('') }}" - channel: "{{ channel | default('*') }}" - - - name: Compute list of years to search - set_fact: - year_range: >- - {{ range( - (starttime | regex_replace('T.*$', '') | to_datetime('%Y-%m-%d')).year, - (endtime | regex_replace('T.*$', '') | to_datetime('%Y-%m-%d')).year + 1 - ) | list }} - - - name: Generate list of -name patterns for DOY matching across years - shell: | - start="{{ starttime | regex_replace('T.*$', '') }}" - end="{{ endtime | regex_replace('T.*$', '') }}" - tmp_list="" - - start_epoch=$(date -d "$start" +%s) - end_epoch=$(date -d "$end" +%s) - days_diff=$(( (end_epoch - start_epoch) / 86400 )) - - for i in $(seq 0 $days_diff); do - current_date=$(date -d "@$((start_epoch + i * 86400))" +%Y.%j) - year=$(echo "$current_date" | cut -d. -f1) - doy=$(echo "$current_date" | cut -d. -f2) - tmp_list="$tmp_list -name '*{{ file_suffix }}.${year}.${doy}' -o" - done - - echo "${tmp_list% -o}" - args: - executable: /bin/bash - register: name_patterns - changed_when: false - - - name: Set final find arguments - set_fact: - find_args: "{{ name_patterns.stdout }}" - - - name: Run find in correct SDS folders per year - shell: > - find iso_sds/{{ item }}/{{ network }}/{{ station }}/{{ channel }}.{{ file_suffix }}/ -type f {{ find_args }} - loop: "{{ year_range }}" - args: - executable: /bin/bash - register: find_results - changed_when: false - ignore_errors: true - - - name: Collect found SDS file paths - set_fact: - all_files: >- - {{ find_results.results | map(attribute='stdout_lines') | flatten | list }} - - - name: Show matched file paths (debug) - debug: - var: all_files - - - name: Show count of matched files - debug: - msg: "Total matched files: {{ all_files | length }}" - - - name: Run seedpsd-cli asynchronously using process substitution - shell: | - cd {{ seedpsd_dir }} && - uv run seedpsd-cli data mseed --simulate --list <( - {% for f in all_files %} - echo {{ f | quote }} - {% endfor %} - ) - args: - executable: /bin/bash - async: 3600 - poll: 0 - register: seedpsd_async - - - name: "Track job: keep only latest seedpsd data job and preserve others" - block: - - - name: Ensure ~/.dmtri/jobs/ directory exists - file: - path: "~/.dmtri/jobs/{{ inventory_hostname }}" - state: directory - mode: '0755' - - - name: Load existing job list (if exists) - slurp: - src: "~/.dmtri/jobs/{{ inventory_hostname }}/seedpsd_data.json" - register: job_file - ignore_errors: true - - - name: Parse job list or default to [] - set_fact: - job_list: >- - {{ (job_file.content | b64decode | from_json) if job_file.content is defined else [] }} - - - name: Define new job entry - set_fact: - new_job: { - "type": "seedpsd", - "subtype": "data", - "job_id": "{{ seedpsd_async.ansible_job_id }}", - "host": "{{ inventory_hostname }}", - "network": "{{ network }}", - "station": "{{ station }}", - "start": "{{ starttime }}", - "end": "{{ endtime }}", - "created_at": "{{ lookup('pipe', 'date -u +%Y-%m-%dT%H:%M:%SZ') }}" - } - - - name: Remove any previous job of the same type and subtype - set_fact: - filtered_jobs: >- - {{ - job_list - | rejectattr('type', 'equalto', new_job.type) - | list - + job_list - | selectattr('type', 'equalto', new_job.type) - | selectattr('subtype', 'defined') - | rejectattr('subtype', 'equalto', new_job.subtype) - | list - }} - - - name: Save updated job list (one job per subtype) - copy: - content: "{{ (filtered_jobs + [new_job]) | to_nice_json }}" - dest: "~/.dmtri/jobs/{{ inventory_hostname }}/seedpsd_data.json" diff --git a/src/playbooks/refresh/wfcatalog_refresh.yml b/src/playbooks/refresh/wfcatalog_refresh.yml deleted file mode 100644 index 368c5a9..0000000 --- a/src/playbooks/refresh/wfcatalog_refresh.yml +++ /dev/null @@ -1,114 +0,0 @@ -- name: Run WFCatalogCollector with custom ARCHIVE_ROOT (async) - hosts: wf_catalogue - gather_facts: false - - vars: - venv_python: ".env/bin/python3.11" - collector_script: "WFCatalogCollector.py" - config_path: "{{ collector_dir }}/config.json" - - tasks: - - - name: Ensure required variables are provided - fail: - msg: "You must pass 'network', 'station', 'starttime', and 'endtime' via -e" - when: network is not defined or station is not defined or starttime is not defined or endtime is not defined - - - name: Compute basic derived values - set_fact: - start_date: "{{ starttime | regex_replace('T.*$', '') }}" - end_date: "{{ endtime | regex_replace('T.*$', '') }}" - year: "{{ starttime[0:4] }}" - - - name: Compute range and paths - set_fact: - range_days: >- - {{ - ( - (endtime | to_datetime('%Y-%m-%dT%H:%M:%S')) - - (starttime | to_datetime('%Y-%m-%dT%H:%M:%S')) - ).days + 1 - }} - log_path: "{{ collector_dir }}/log/WFCatalog-{{ network }}-{{ station }}-{{ start_date }}-{{ end_date }}.log" - custom_archive_root: "/home/sysop/iso_sds/{{ year }}/{{ network }}/{{ station }}" - - - name: Check if backup exists - stat: - path: "{{ config_path }}.back" - register: config_backup - - - name: Backup original config.json - copy: - src: "{{ config_path }}" - dest: "{{ config_path }}.back" - remote_src: true - when: not config_backup.stat.exists - - - name: Update ARCHIVE_ROOT in config.json - replace: - path: "{{ config_path }}" - regexp: '"ARCHIVE_ROOT":\s*".*?"' - replace: '"ARCHIVE_ROOT": "{{ custom_archive_root }}"' - - - name: Run WFCatalogCollector asynchronously - shell: | - cd {{ collector_dir }} && \ - {{ venv_python }} {{ collector_script }} \ - --flags --csegs \ - --logfile {{ log_path }} \ - --date {{ start_date }} \ - --range {{ range_days }} \ - --update --stdout - args: - executable: /bin/bash - async: 3600 - poll: 0 - register: wfcatalog_async - - - name: "Track job: keep only latest 'wfcatalog' job and preserve others" - block: - - - name: Ensure ~/.dmtri/jobs/{{ inventory_hostname }} directory exists - file: - path: "~/.dmtri/jobs/{{ inventory_hostname }}" - state: directory - mode: '0755' - - - name: Load existing job list (if exists) - slurp: - src: "~/.dmtri/jobs/{{ inventory_hostname }}/wfcatalog.json" - register: job_file - ignore_errors: true - - - name: Parse job list or default to [] - set_fact: - job_list: >- - {{ (job_file.content | b64decode | from_json) if job_file.content is defined else [] }} - - - name: Define new job entry - set_fact: - new_job: { - "type": "wfcatalog", - "job_id": "{{ wfcatalog_async.ansible_job_id }}", - "host": "{{ inventory_hostname }}", - "network": "{{ network }}", - "station": "{{ station }}", - "start": "{{ starttime }}", - "end": "{{ endtime }}", - "created_at": "{{ lookup('pipe', 'date -u +%Y-%m-%dT%H:%M:%SZ') }}" - } - - - name: Remove any previous job of the same type - set_fact: - filtered_jobs: "{{ job_list | rejectattr('type', 'equalto', new_job.type) | list }}" - - - name: Save updated job list (one per type) - copy: - content: "{{ (filtered_jobs + [new_job]) | to_nice_json }}" - dest: "~/.dmtri/jobs/{{ inventory_hostname }}/wfcatalog.json" - - - name: Restore original config.json - copy: - src: "{{ config_path }}.back" - dest: "{{ config_path }}" - remote_src: true diff --git a/src/playbooks/track.yml b/src/playbooks/track.yml deleted file mode 100644 index 850083d..0000000 --- a/src/playbooks/track.yml +++ /dev/null @@ -1,91 +0,0 @@ -- name: "Track all dmtri async jobs across all hosts" - hosts: all - gather_facts: false - - tasks: - - - name: "Gather remote environment variables" - setup: - filter: ansible_env - - - name: "Find job tracking files in remote ~/.dmtri/jobs/{{ inventory_hostname }}/" - find: - paths: "{{ ansible_env.HOME }}/.dmtri/jobs/{{ inventory_hostname }}" - patterns: "*.json" - file_type: file - register: job_files - ignore_errors: true - - - name: "Fail if no job files were found on this host" - fail: - msg: "No job tracking files found for {{ inventory_hostname }}" - when: job_files.files | length == 0 - - - name: "Load job files" - slurp: - src: "{{ item.path }}" - loop: "{{ job_files.files }}" - register: loaded_job_data - loop_control: - label: "{{ item.path }}" - - - name: "Parse and combine job lists" - set_fact: - combined_jobs: >- - {{ - loaded_job_data.results - | map(attribute='content') - | map('b64decode') - | map('from_json') - | flatten - }} - - - name: "Check async job statuses (IDs starting with j)" - async_status: - jid: "{{ item.job_id }}" - loop: "{{ combined_jobs | selectattr('job_id', 'match', '^j\\d+') | list }}" - register: job_statuses - loop_control: - label: "{{ item.job_id }}" - - - name: "Build manual job results (IDs like manual-xxxx)" - set_fact: - manual_jobs: >- - {{ - combined_jobs - | selectattr('job_id', 'match', '^manual-') - | map('combine', {'status': '✅ Manual (not tracked)'}) - | list - }} - - - name: "Merge async and manual job results" - set_fact: - all_job_results: >- - {{ - (job_statuses.results | default([])) + manual_jobs - }} - - - name: "Print concise job summary per host" - debug: - msg: >- - {{ '%-18s' | format(item.item.host | default(inventory_hostname)) }} - {{ '%-28s' | format(item.item.job_id if item.item is defined else item.job_id) }} - {{ '%-22s' | format((item.item.type if item.item is defined else item.type) ~ - (' (' ~ (item.item.subtype if item.item is defined else item.subtype) ~ ')' - if (item.item.subtype if item.item is defined else item.subtype) is defined else '')) }} - {{ '%-10s' | format(item.item.network | default('—')) }} - {{ '%-10s' | format(item.item.station | default('—')) }} - {{ '%-20s' | format(item.item.start | default('—')) }} - {{ '%-20s' | format(item.item.end | default('—')) }} - {{ ( - item.status | default( - item.started is defined and item.started | bool | - ternary( - item.finished | default(false) | bool | ternary('✅ Finished', '⏳ Running'), - '❌ Failed' - ) - ) - ) }} - loop: "{{ all_job_results }}" - loop_control: - label: "{{ item.item.job_id if item.item is defined else item.job_id }}" diff --git a/tests/test_cli.py b/tests/test_cli.py index 9c6bfd3..8963095 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,175 +1,184 @@ -import pytest -from datetime import datetime, timedelta, timezone -from unittest.mock import patch -from dmtri import cli -import sys - - -def test_cli_version_flag(capsys): - with pytest.raises(SystemExit): - with patch("sys.argv", ["dmtri", "--version"]): - cli.main() - captured = capsys.readouterr() - assert "version" in captured.out.lower() - - -def test_cli_no_command_prints_help(capsys): - with pytest.raises(SystemExit) as e: - with patch("sys.argv", ["dmtri"]): - cli.main() - assert e.value.code == 1 - captured = capsys.readouterr() - assert "usage" in captured.out.lower() - -@patch("dmtri.cli.run_hook") -def test_cli_refresh_no_confirm_runs(mock_run_hook): - with patch("builtins.input", return_value="y"): - args = [ - "dmtri", "refresh", - "--network", "GR", - "--station", "ATH", - "--starttime", "2025-01-01T00:00:00", - "--no-confirm" - ] - with patch("sys.argv", args): - cli.main() - - assert mock_run_hook.call_count == 4 - - -@patch("dmtri.cli.run_hook") -@patch("dmtri.cli.print_execution_plan") -def test_cli_refresh_with_confirm_runs(mock_print_plan, mock_run_hook): - with patch("builtins.input", return_value="y"): - args = [ - "dmtri", "refresh", - "--network", "GR", - "--station", "ATH", - "--starttime", "2025-01-01T00:00:00" - ] - with patch("sys.argv", args): - cli.main() - - assert mock_print_plan.call_count == 4 - assert mock_run_hook.call_count == 4 - - - -def test_cli_requires_starttime(): - args = ["dmtri", "refresh", "--network", "GR", "--station", "ATH"] - with patch("sys.argv", args): - with pytest.raises(SystemExit) as e: - cli.main() - assert e.value.code == 2 - - -def test_cli_invalid_date_format(): - args = ["dmtri", "refresh", "--network", "GR", "--station", "ATH", "--starttime", "bad-date"] - with patch("sys.argv", args): - with pytest.raises(SystemExit) as e: - cli.main() - assert e.value.code == 1 - - -@patch("builtins.input", return_value="y") -def test_cli_invalid_type_rejected(mock_input): - args = [ - "dmtri", "refresh", - "--network", "GR", "--station", "ATH", - "--starttime", "2025-01-01T00:00:00", - "--endtime", "2025-04-01T00:00:00", - "--type", "test" - ] - with patch("sys.argv", args): - with pytest.raises(SystemExit) as e: - cli.main() - assert e.value.code == 1 - - - -def test_cli_starttime_in_future_rejected(): - future = (datetime.now(timezone.utc) + timedelta(days=2)).isoformat() - args = ["dmtri", "refresh", "--network", "GR", "--station", "ATH", "--starttime", future] - with patch("sys.argv", args): - with pytest.raises(SystemExit) as e: - cli.main() - assert e.value.code == 1 - - -@patch("builtins.input", return_value="n") -def test_cli_user_aborts_on_long_range(mock_input, capsys): - args = [ - "dmtri", "refresh", - "--network", "GR", - "--station", "ATH", - "--starttime", "2025-01-01T00:00:00", - "--endtime", "2025-01-20T00:00:00" - ] - with patch("sys.argv", args): - with pytest.raises(SystemExit) as e: - cli.main() - assert e.value.code == 0 - - out = capsys.readouterr().out - assert "aborted by user" in out.lower() - -@patch("dmtri.doctor.run_diagnostics") -def test_cli_doctor_runs(mock_run): - with patch("sys.argv", ["dmtri", "doctor"]): - cli.main() - mock_run.assert_called_once_with(verbose=False) - -@patch("dmtri.doctor.run_diagnostics") -def test_cli_doctor_verbose(mock_run): - with patch("sys.argv", ["dmtri", "doctor", "--verbose"]): - cli.main() - mock_run.assert_called_once_with(verbose=True) -def test_cli_aborts_on_user_input(monkeypatch): - test_args = [ - "dmtri", "refresh", - "--network", "FR", - "--starttime", "2024-01-01", - "--endtime", "2024-01-02" - ] - monkeypatch.setattr(sys, "argv", test_args) - - with patch("builtins.input", return_value="n"): - with pytest.raises(SystemExit) as excinfo: - cli.main() - assert excinfo.value.code == 0 # Means user aborted -@patch("builtins.input", return_value="y") # Prevent blocking on input() -def test_cli_exits_if_no_playbook_for_command(mock_input, monkeypatch): - test_args = [ - "dmtri", "refresh", - "--network", "FR", - "--starttime", "2024-01-01" - ] - monkeypatch.setattr(sys, "argv", test_args) - - # Temporarily remove any configured playbooks for 'refresh' - from dmtri.cli import COMMAND_PLAYBOOKS - original_playbooks = COMMAND_PLAYBOOKS.get("refresh") - COMMAND_PLAYBOOKS["refresh"] = {} - - try: - with pytest.raises(SystemExit) as e: - cli.main() - assert e.value.code == 1 - finally: - # Restore after test - COMMAND_PLAYBOOKS["refresh"] = original_playbooks -@patch("builtins.input", side_effect=["y", "n"]) -@patch("dmtri.cli.print_execution_plan") -def test_cli_user_aborts_after_plan(mock_print_plan, mock_input, monkeypatch): - args = [ - "dmtri", "refresh", - "--network", "GR", - "--station", "ATH", - "--starttime", "2025-01-01T00:00:00" - ] - monkeypatch.setattr(sys, "argv", args) - - with pytest.raises(SystemExit) as e: - cli.main() - - assert e.value.code == 0 +import pytest +from datetime import datetime, timedelta, timezone +from unittest.mock import patch +from dmtri import cli +import sys +from pathlib import Path + + +@pytest.fixture(autouse=True) +def mock_inventory_path(monkeypatch): + """Mock get_inventory_path to return a dummy path during tests.""" + monkeypatch.setattr("dmtri.cli.get_inventory_path", lambda override=None: Path("/tmp/hosts.ini")) + + +def test_cli_version_flag(capsys): + with pytest.raises(SystemExit): + with patch("sys.argv", ["dmtri", "--version"]): + cli.main() + captured = capsys.readouterr() + assert "version" in captured.out.lower() + + +def test_cli_no_command_prints_help(capsys): + with pytest.raises(SystemExit) as e: + with patch("sys.argv", ["dmtri"]): + cli.main() + assert e.value.code == 1 + captured = capsys.readouterr() + assert "usage" in captured.out.lower() + +@patch("dmtri.cli.run_hook") +def test_cli_refresh_no_confirm_runs(mock_run_hook): + with patch("builtins.input", return_value="y"): + args = [ + "dmtri", "refresh", + "--network", "GR", + "--station", "ATH", + "--starttime", "2025-01-01T00:00:00", + "--no-confirm" + ] + with patch("sys.argv", args): + cli.main() + + assert mock_run_hook.call_count == 4 + + +@patch("dmtri.cli.run_hook") +@patch("dmtri.cli.print_execution_plan") +def test_cli_refresh_with_confirm_runs(mock_print_plan, mock_run_hook): + with patch("builtins.input", return_value="y"): + args = [ + "dmtri", "refresh", + "--network", "GR", + "--station", "ATH", + "--starttime", "2025-01-01T00:00:00" + ] + with patch("sys.argv", args): + cli.main() + + assert mock_print_plan.call_count == 4 + assert mock_run_hook.call_count == 4 + + + +def test_cli_requires_starttime(): + args = ["dmtri", "refresh", "--network", "GR", "--station", "ATH"] + with patch("sys.argv", args): + with pytest.raises(SystemExit) as e: + cli.main() + assert e.value.code == 2 + + +def test_cli_invalid_date_format(): + args = ["dmtri", "refresh", "--network", "GR", "--station", "ATH", "--starttime", "bad-date"] + with patch("sys.argv", args): + with pytest.raises(SystemExit) as e: + cli.main() + assert e.value.code == 2 + + +@patch("builtins.input", return_value="y") +def test_cli_invalid_type_rejected(mock_input): + args = [ + "dmtri", "refresh", + "--network", "GR", "--station", "ATH", + "--starttime", "2025-01-01T00:00:00", + "--endtime", "2025-04-01T00:00:00", + "--type", "test" + ] + with patch("sys.argv", args): + with pytest.raises(SystemExit) as e: + cli.main() + assert e.value.code == 1 + + + +def test_cli_starttime_in_future_rejected(): + future = (datetime.now(timezone.utc) + timedelta(days=2)).isoformat() + args = ["dmtri", "refresh", "--network", "GR", "--station", "ATH", "--starttime", future] + with patch("sys.argv", args): + with pytest.raises(SystemExit) as e: + cli.main() + assert e.value.code == 2 + + +@patch("builtins.input", return_value="n") +def test_cli_user_aborts_on_long_range(mock_input, capsys): + args = [ + "dmtri", "refresh", + "--network", "GR", + "--station", "ATH", + "--starttime", "2025-01-01T00:00:00", + "--endtime", "2025-01-20T00:00:00" + ] + with patch("sys.argv", args): + with pytest.raises(SystemExit) as e: + cli.main() + assert e.value.code == 0 + + out = capsys.readouterr().out + assert "aborted by user" in out.lower() + +@patch("dmtri.doctor.run_diagnostics") +def test_cli_doctor_runs(mock_run): + with patch("sys.argv", ["dmtri", "doctor"]): + cli.main() + from unittest.mock import ANY + mock_run.assert_called_once_with(verbose=False, inventory=ANY) + +@patch("dmtri.doctor.run_diagnostics") +def test_cli_doctor_verbose(mock_run): + with patch("sys.argv", ["dmtri", "doctor", "--verbose"]): + cli.main() + from unittest.mock import ANY + mock_run.assert_called_once_with(verbose=True, inventory=ANY) +def test_cli_aborts_on_user_input(monkeypatch): + test_args = [ + "dmtri", "refresh", + "--network", "FR", + "--starttime", "2024-01-01", + "--endtime", "2024-01-02" + ] + monkeypatch.setattr(sys, "argv", test_args) + + with patch("builtins.input", return_value="n"): + with pytest.raises(SystemExit) as excinfo: + cli.main() + assert excinfo.value.code == 0 # Means user aborted +@patch("builtins.input", return_value="y") # Prevent blocking on input() +def test_cli_exits_if_no_playbook_for_command(mock_input, monkeypatch): + test_args = [ + "dmtri", "refresh", + "--network", "FR", + "--starttime", "2024-01-01" + ] + monkeypatch.setattr(sys, "argv", test_args) + + # Temporarily remove any configured playbooks for 'refresh' + from dmtri.cli import COMMAND_PLAYBOOKS + original_playbooks = COMMAND_PLAYBOOKS.get("refresh") + COMMAND_PLAYBOOKS["refresh"] = {} + + try: + with pytest.raises(SystemExit) as e: + cli.main() + assert e.value.code == 1 + finally: + # Restore after test + COMMAND_PLAYBOOKS["refresh"] = original_playbooks +@patch("builtins.input", side_effect=["y", "n"]) +@patch("dmtri.cli.print_execution_plan") +def test_cli_user_aborts_after_plan(mock_print_plan, mock_input, monkeypatch): + args = [ + "dmtri", "refresh", + "--network", "GR", + "--station", "ATH", + "--starttime", "2025-01-01T00:00:00" + ] + monkeypatch.setattr(sys, "argv", args) + + with pytest.raises(SystemExit) as e: + cli.main() + + assert e.value.code == 0 diff --git a/tests/test_doctor.py b/tests/test_doctor.py index 1f29cae..1d56412 100644 --- a/tests/test_doctor.py +++ b/tests/test_doctor.py @@ -1,53 +1,53 @@ -import pytest -from unittest.mock import patch, MagicMock -from dmtri import doctor - - -@patch("ansible_runner.run") -def test_run_diagnostics_all_ok(mock_run, caplog): - mock_event = {"event": "runner_on_ok", "event_data": {"host": "10.0.0.248"}} - mock_run.return_value.events = [mock_event] - - with caplog.at_level("INFO"): - doctor.run_diagnostics(verbose=False) - - assert "[OK] 10.0.0.248" in caplog.text - - - -@patch("ansible_runner.run") -def test_run_diagnostics_unreachable(mock_run, caplog): - mock_event = {"event": "runner_on_unreachable", "event_data": {"host": "10.0.0.248"}} - mock_run.return_value.events = [mock_event] - - doctor.run_diagnostics(verbose=False) - - out = caplog.text - assert "[UNREACHABLE] 10.0.0.248" in out - - -@patch("ansible_runner.run") -def test_run_diagnostics_no_response(mock_run, caplog): - mock_run.return_value.events = [] - - doctor.run_diagnostics(verbose=False) - - out = caplog.text - assert "No responses received" in out -@patch("ansible_runner.run") -def test_run_diagnostics_skips_non_dict_and_missing_host(mock_run, caplog): - # Non-dict event - bad_event1 = "not a dict" - - # Event without host - bad_event2 = {"event": "runner_on_ok", "event_data": {}} - - # Good event for sanity check - good_event = {"event": "runner_on_ok", "event_data": {"host": "10.0.0.248"}} - - mock_run.return_value.events = [bad_event1, bad_event2, good_event] - - with caplog.at_level("INFO"): - doctor.run_diagnostics(verbose=False) - - assert "[OK] 10.0.0.248" in caplog.text +import pytest +from unittest.mock import patch, MagicMock +from dmtri import doctor + + +@patch("ansible_runner.run") +def test_run_diagnostics_all_ok(mock_run, caplog): + mock_event = {"event": "runner_on_ok", "event_data": {"host": "10.0.0.248"}} + mock_run.return_value.events = [mock_event] + + with caplog.at_level("INFO"): + doctor.run_diagnostics(verbose=False) + + assert "[OK] 10.0.0.248" in caplog.text + + + +@patch("ansible_runner.run") +def test_run_diagnostics_unreachable(mock_run, caplog): + mock_event = {"event": "runner_on_unreachable", "event_data": {"host": "10.0.0.248"}} + mock_run.return_value.events = [mock_event] + + doctor.run_diagnostics(verbose=False) + + out = caplog.text + assert "[UNREACHABLE] 10.0.0.248" in out + + +@patch("ansible_runner.run") +def test_run_diagnostics_no_response(mock_run, caplog): + mock_run.return_value.events = [] + + doctor.run_diagnostics(verbose=False) + + out = caplog.text + assert "No responses received" in out +@patch("ansible_runner.run") +def test_run_diagnostics_skips_non_dict_and_missing_host(mock_run, caplog): + # Non-dict event + bad_event1 = "not a dict" + + # Event without host + bad_event2 = {"event": "runner_on_ok", "event_data": {}} + + # Good event for sanity check + good_event = {"event": "runner_on_ok", "event_data": {"host": "10.0.0.248"}} + + mock_run.return_value.events = [bad_event1, bad_event2, good_event] + + with caplog.at_level("INFO"): + doctor.run_diagnostics(verbose=False) + + assert "[OK] 10.0.0.248" in caplog.text diff --git a/tests/test_utils.py b/tests/test_utils.py index 574bef2..665fe50 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,6 +2,8 @@ from argparse import Namespace, ArgumentParser from datetime import datetime, timedelta, timezone from unittest.mock import patch +from dmtri.utils import parse_flexible_date +from types import SimpleNamespace from dmtri.utils import ( get_version, maybe_warn_shell_globbing, @@ -19,17 +21,15 @@ def test_get_version_reads_pyproject(): def test_parse_datetime_args_valid(): - args = Namespace(starttime="2025-01-01T00:00:00", endtime="2025-01-02T00:00:00") + args = SimpleNamespace( + starttime=parse_flexible_date("2025-01-01T00:00:00"), + endtime=parse_flexible_date("2025-01-02T00:00:00") + ) start, end = parse_datetime_args(args) assert start.isoformat().startswith("2025-01-01") assert end.isoformat().startswith("2025-01-02") -def test_parse_datetime_args_invalid(): - args = Namespace(starttime="not-a-date", endtime="2025-01-02T00:00:00") - with pytest.raises(SystemExit): - parse_datetime_args(args) - def test_validate_datetime_range_valid(): now = datetime.now(timezone.utc) @@ -86,10 +86,13 @@ def test_validate_and_normalize_args_valid(mock_input): args = Namespace( command="refresh", network=["GR"], station=["ATH"], location=["*"], channel=["*"], - starttime="2025-01-01T00:00:00", endtime="2025-01-02T00:00:00", + starttime=parse_flexible_date("2025-01-01T00:00:00"), + endtime=parse_flexible_date("2025-01-02T00:00:00"), type="data" ) start, end, types = validate_and_normalize_args(args, parser) + assert isinstance(start, datetime) + assert isinstance(end, datetime) assert start < end assert types == ["data"]