From 4886ce07a90c32d51127e55a58493753d90d692e Mon Sep 17 00:00:00 2001 From: Mohit Bisht Date: Wed, 2 Sep 2026 21:25:44 +0530 Subject: [PATCH 01/12] Add Docker development environment for hackathon --- .dockerignore | 49 +++++++++++++------------------------------- Dockerfile | 35 +++++++++++-------------------- HACKATHON.md | 20 ++++++++++++++++++ compose.yaml | 15 ++++++++++++++ docker/entrypoint.sh | 9 ++++++++ 5 files changed, 70 insertions(+), 58 deletions(-) create mode 100644 HACKATHON.md create mode 100644 compose.yaml create mode 100644 docker/entrypoint.sh diff --git a/.dockerignore b/.dockerignore index 185fd7fde..34350adec 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,36 +1,15 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +.git +.gitignore -* -!Dockerfile -!python/ -python/** -!python/Dockerfile -!python/docker/ -!python/docker/** -!python/examples/ -!python/examples/** -!python/pyproject.toml -!python/setup.cfg -!python/README.md -!python/src/ -python/src/** -!python/src/pywy/ -!python/src/pywy/** -!wayang-assembly/ -wayang-assembly/** -!wayang-assembly/target/ -wayang-assembly/target/** -!wayang-assembly/target/*-dist.tar.gz +**/target/ + +.idea/ +.vscode/ +*.iml + +*.log + +.DS_Store +Thumbs.db + +.m2/ diff --git a/Dockerfile b/Dockerfile index b75a5b82c..95e9718fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,33 +13,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM eclipse-temurin:17-jre AS wayang-assembler +FROM eclipse-temurin:17-jdk -ARG WAYANG_DIST=wayang-assembly/target/*-dist.tar.gz +WORKDIR /workspace/wayang -COPY ${WAYANG_DIST} /tmp/wayang-dist.tar.gz +RUN apt-get update \ + && apt-get install -y --no-install-recommends git bash \ + && rm -rf /var/lib/apt/lists/* -RUN mkdir -p /opt/wayang-root /opt/wayang /opt/wayang/smoke \ - && tar -xzf /tmp/wayang-dist.tar.gz -C /opt/wayang-root \ - && extracted_dir="$(find /opt/wayang-root -mindepth 1 -maxdepth 1 -type d -name 'wayang-*' | head -n 1)" \ - && test -n "${extracted_dir}" \ - && cp -a "${extracted_dir}/." /opt/wayang/ \ - && find /opt/wayang -type f \( -name "*-sources.jar" -o -name "*-javadoc.jar" -o -name "README.md" \) -delete \ - && printf "apache wayang docker smoke test\n" > /opt/wayang/smoke/wordcount.txt \ - && rm /tmp/wayang-dist.tar.gz +COPY . /workspace/wayang -FROM eclipse-temurin:17-jre +RUN sed -i 's/\r$//' mvnw \ + && chmod +x mvnw -ENV WAYANG_HOME=/opt/wayang -ENV FLAG_WAYANG=true -ENV PATH="${WAYANG_HOME}/bin:${PATH}" +COPY docker/entrypoint.sh /usr/local/bin/wayang-entrypoint -COPY --from=wayang-assembler /opt/wayang /opt/wayang +RUN chmod +x /usr/local/bin/wayang-entrypoint -RUN mkdir -p /root/.wayang /opt/wayang/conf \ - && touch /opt/wayang/conf/wayang.properties - -WORKDIR /opt/wayang - -ENTRYPOINT ["/opt/wayang/bin/wayang-submit"] -CMD ["org.apache.wayang.apps.pi.PiEstimation", "java", "1"] +ENTRYPOINT ["wayang-entrypoint"] +CMD ["bash"] \ No newline at end of file diff --git a/HACKATHON.md b/HACKATHON.md new file mode 100644 index 000000000..ce6e94045 --- /dev/null +++ b/HACKATHON.md @@ -0,0 +1,20 @@ +# Wayang Hackathon Development Environment + +This guide provides a Docker-based development environment for the Wayang +hackathon. + +The goal is to allow participants to get a working Wayang environment without +having to install Java, Maven, Spark, Hadoop, or other Wayang dependencies +locally. + +## Prerequisites + +Install: + +- Git +- Docker Desktop + +Verify Docker is available: + +```bash +docker --version \ No newline at end of file diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 000000000..4f5da9f26 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,15 @@ +services: + wayang: + build: + context: . + dockerfile: Dockerfile + container_name: wayang-hackathon + working_dir: /workspace/wayang + volumes: + - .:/workspace/wayang + - wayang-maven-cache:/root/.m2 + stdin_open: true + tty: true + +volumes: + wayang-maven-cache: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100644 index 000000000..c4a15bb09 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -e + +if [ -f /workspace/wayang/mvnw ]; then + sed -i 's/\r$//' /workspace/wayang/mvnw + chmod +x /workspace/wayang/mvnw +fi + +exec "$@" \ No newline at end of file From df9d0c75d73c65944748d30a9c0632698d83dfb9 Mon Sep 17 00:00:00 2001 From: Mohit Bisht Date: Mon, 28 Sep 2026 22:09:02 +0530 Subject: [PATCH 02/12] Add hackathon Docker development environment --- .github/workflows/hackathon-image.yml | 69 ++++++ hackathon-CoC2026/Dockerfile | 72 +++++++ hackathon-CoC2026/README.md | 199 ++++++++++++++++++ hackathon-CoC2026/compose.yaml | 80 +++++++ hackathon-CoC2026/docker/entrypoint.sh | 28 +++ hackathon-CoC2026/docker/postgres/init.sql | 32 +++ .../examples/PostgresSmokeTest.java | 85 ++++++++ hackathon-CoC2026/verify.sh | 80 +++++++ 8 files changed, 645 insertions(+) create mode 100644 .github/workflows/hackathon-image.yml create mode 100644 hackathon-CoC2026/Dockerfile create mode 100644 hackathon-CoC2026/README.md create mode 100644 hackathon-CoC2026/compose.yaml create mode 100644 hackathon-CoC2026/docker/entrypoint.sh create mode 100644 hackathon-CoC2026/docker/postgres/init.sql create mode 100644 hackathon-CoC2026/examples/PostgresSmokeTest.java create mode 100644 hackathon-CoC2026/verify.sh diff --git a/.github/workflows/hackathon-image.yml b/.github/workflows/hackathon-image.yml new file mode 100644 index 000000000..19cefe437 --- /dev/null +++ b/.github/workflows/hackathon-image.yml @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Builds hackathon-CoC2026/Dockerfile (which bakes in a warm Maven cache, +# see that file's comments) and publishes it to GitHub Container Registry. +# +# Deliberately manual (workflow_dispatch) rather than on every push: this +# image is for a specific hackathon date, doesn't need to track every +# commit to main, and a full build is expensive. Re-run manually whenever +# the environment needs to pick up new commits before the event. +# +# NOTE: publishing to ghcr.io/apache/... requires GHCR to be enabled for +# this repository under the ASF GitHub org. If this workflow fails on the +# push step with a permissions error, that's an ASF INFRA setting, not a +# problem with this workflow — check with infra or ask in the PR. + +name: Publish hackathon Docker image + +on: + workflow_dispatch: {} + +env: + IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/wayang-hackathon + +permissions: + contents: read + packages: write + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + file: hackathon-CoC2026/Dockerfile + push: true + tags: | + ${{ env.IMAGE_NAME }}:latest + ${{ env.IMAGE_NAME }}:${{ github.run_number }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/hackathon-CoC2026/Dockerfile b/hackathon-CoC2026/Dockerfile new file mode 100644 index 000000000..2897bb7d1 --- /dev/null +++ b/hackathon-CoC2026/Dockerfile @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Hackathon developer environment (Community over Code 2026, Glasgow). +# +# This is a JDK-based development container, not a slim runtime image: +# participants are expected to edit Wayang source and rebuild Wayang during +# the hackathon, so we keep Maven Wrapper + git + the full source tree +# available inside it. +# +# Build context is the REPO ROOT (see compose.yaml: `context: ..`), because +# the whole source tree needs to be present to build Wayang. + +FROM eclipse-temurin:17-jdk + +LABEL org.opencontainers.image.source="https://github.com/apache/wayang" + +# git -> some mvnw/plugin steps and contributor workflows expect it +# bash -> mvnw and our entrypoint are bash scripts +# postgresql-client -> lets participants run `psql` from inside the +# container to poke at the hackathon Postgres service +RUN apt-get update \ + && apt-get install -y --no-install-recommends git bash postgresql-client \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /workspace/wayang + +# The source tree is normally supplied at runtime via the bind mount in +# compose.yaml (so local edits show up immediately). We still COPY it here +# so `docker build` works standalone, without compose, e.g. in CI. +COPY . /workspace/wayang + +# The repo's Maven Wrapper is frequently checked out with CRLF line endings +# on Windows, which breaks execution inside the Linux container. +RUN sed -i 's/\r$//' mvnw \ + && chmod +x mvnw + +COPY hackathon-CoC2026/docker/entrypoint.sh /usr/local/bin/wayang-entrypoint +RUN chmod +x /usr/local/bin/wayang-entrypoint + +# Pre-warm the Maven local repo (~/.m2) at image build time, so a PUSHED, +# PULLED image already has every dependency downloaded — this is what +# actually gets participants under the 10-15 minute target, vs. a +# from-source build which takes much longer on a cold cache (see +# README.md "Timing"). This step builds against whatever source was +# checked out when this image was built. +# +# ~/.m2 lives outside /workspace/wayang, so it is NOT overwritten by the +# bind mount in compose.yaml. The first time the wayang-maven-cache named +# volume is created, Docker seeds it from this image layer automatically +# (standard behavior for a volume mounted over a non-empty image +# directory), so the warm cache survives into the running container. +RUN ./mvnw -B -q clean package -pl :wayang-assembly -Pdistribution -DskipTests \ + && ./mvnw -B -q dependency:build-classpath -pl :wayang-assembly \ + -Dmdep.outputFile=/tmp/wayang-classpath.txt \ + && rm -rf /workspace/wayang/wayang-assembly/target + +ENTRYPOINT ["wayang-entrypoint"] +CMD ["bash"] diff --git a/hackathon-CoC2026/README.md b/hackathon-CoC2026/README.md new file mode 100644 index 000000000..54802500d --- /dev/null +++ b/hackathon-CoC2026/README.md @@ -0,0 +1,199 @@ +# Wayang Hackathon Development Environment — Community over Code 2026, Glasgow + +This folder is a self-contained Docker development environment for the Wayang +hackathon. It gets a contributor from a fresh checkout to a working build of +Wayang with the **Java**, **Spark**, and **PostgreSQL** platforms all +runnable inside the container. + +This is a *developer* environment, not just a runtime: the full source tree +is bind-mounted into the container, so you edit code on your host machine +with your usual editor/IDE, and build/run it inside the container. Nothing +beyond Git and Docker Desktop is required on the host. + +**Timing**: building the Wayang distribution from source on a cold Maven +cache takes 30-50+ minutes depending on network speed and machine specs — +that's just how long compiling Wayang and its dependency tree (including +Spark) genuinely takes the first time. That is NOT what hackathon +participants are expected to do. Instead, the image is built and pushed to +GitHub Container Registry ahead of the event (see `.github/workflows/ +hackathon-image.yml`), with its Maven cache already warmed at build time. +Participants just pull it: + +```bash +docker compose pull +docker compose up -d +``` + +This is the intended 10-15 minute setup path — it's a Docker image pull +plus container startup, not a build. `docker compose up -d --build` +(building from source locally) is still available and is what this README +documents step by step below, for contributors actively working on the +Dockerfile/build itself — participants on hackathon day shouldn't need it. + +## What's here + +``` +hackathon-CoC2026/ +├── Dockerfile # JDK 17 + git + Maven Wrapper + psql client +├── compose.yaml # the "wayang" dev container + a real "postgres" service +├── verify.sh # one-command build + cross-platform verification +├── docker/ +│ ├── entrypoint.sh # normalizes mvnw line endings on every start +│ └── postgres/ +│ └── init.sql # seeds a small table used by the Postgres smoke test +├── examples/ +│ └── PostgresSmokeTest.java # a Wayang plan spanning Java, Spark, and Postgres +└── README.md # this file +``` + +## Prerequisites + +Install on your host machine: + +- [Git](https://git-scm.com/) +- [Docker Desktop](https://www.docker.com/products/docker-desktop/) (includes + Docker Compose) + +Verify: + +```bash +docker --version +docker compose version +``` + +## 1. Get the source + +```bash +git clone https://github.com/apache/wayang.git +cd wayang +``` + +(If you're working from a fork/branch, clone that instead.) + +## 2. Start the environment + +From this folder: + +```bash +cd hackathon-CoC2026 +docker compose pull # gets the pre-built, cache-warmed image (fast) +docker compose up -d +``` + +If you're actively changing the Dockerfile or want a from-source build +instead, use `docker compose up -d --build` (see Timing above — this is +slow on a cold cache). + +Either way, this starts two services: + +- `wayang` — the container you'll work in (Java 17 JDK, Maven Wrapper, git) +- `postgres` — a real PostgreSQL 16 instance, seeded with a small sample + table (`word_counts_seed`) + +Check both containers are up: + +```bash +docker compose ps +``` + +`postgres` should show as `healthy` before moving on (it has a healthcheck +that waits for it to accept connections). + +## 3. Run the verification script + +```bash +docker compose exec wayang bash hackathon-CoC2026/verify.sh +``` +**Note:** `verify.sh` performs a full Wayang distribution build to validate +the development environment end-to-end. This build can take 30-50+ minutes +on a cold Maven cache. It is not required to start the pre-built hackathon +environment and is not part of the 10-15 minute participant setup target. + +This single command: builds `wayang-assembly`, extracts the distribution, +runs the bundled WordCount app on the Java platform, resolves the full +Maven runtime classpath, and compiles + runs `PostgresSmokeTest.java` — a +Wayang plan that reads from Postgres, filters on Java, and maps on Spark. + +The distribution's extracted layout is `bin/`, `conf/`, `jars/`, and +`libs/`. Both `jars/*` and `libs/*` are needed on the classpath to compile +and run against Wayang's own modules — `jars/` alone is not enough. Neither +directory contains Spark's own transitive runtime dependencies (e.g. +`spark-sql`), though — those are only resolved through Maven's local +repository, via `./mvnw -pl wayang-assembly dependency:build-classpath`, +which is why `verify.sh` builds the classpath from both sources combined. + +You do not need to hand-edit `bin/wayang-submit` or add `--add-opens`/ +`--add-exports` flags yourself — the script uses `wayang-submit` as-is for +the Java check, and pulls the same flags straight out of that script for its +own direct `java` invocation of the smoke test. + +Confirmed working output looks like this: + +``` +Wayang plan executed across Postgres, Java, and Spark. Results: +WAYANG -> 42 +SPARK -> 17 +POSTGRES -> 9 +Total rows: 3 +``` + +If it fails, see Troubleshooting below. + +You can also poke at the database directly from inside the container: + +```bash +psql -h postgres -U wayang -d wayang_hackathon -c "select * from word_counts_seed;" +# password: wayang +``` + +## Troubleshooting + +- **`mvnw: command not found` / permission denied** — the entrypoint fixes + line endings and permissions on `mvnw` on every container start, but if + you edited/replaced it from the host afterward, re-run + `sed -i 's/\r$//' mvnw && chmod +x mvnw` inside the container. +- **Postgres connection refused** — wait for `docker compose ps` to show + `postgres` as `healthy`; the `wayang` service is configured to start after + that, but a manual `docker compose exec` shell you opened earlier may have + started before it. Just retry. +- **Slow first build** — this is the Maven dependency download, not a bug + (see Timing above). It's cached in the `wayang-maven-cache` volume; + `docker compose down` (without `-v`) keeps it, `docker compose down -v` + deletes it. +- **Out of memory during the build** — increase Docker Desktop's memory + limit (Settings → Resources). +- **`ClassNotFoundException`/`NoClassDefFoundError` for a Spark class** — + confirm `/tmp/wayang-classpath.txt` from `verify.sh` actually contains + `spark-sql` and `spark-sql-api` jars; if not, re-run the `dependency: + build-classpath` step and check for Maven resolution errors above it. + +## Resetting everything + +```bash +docker compose down -v +``` + +This removes both containers and both named volumes (Maven cache and +Postgres data), so the next `docker compose up -d --build` starts completely +fresh. + +## For maintainers: publishing the image + +`.github/workflows/hackathon-image.yml` builds `hackathon-CoC2026/Dockerfile` +and pushes it to `ghcr.io/apache/wayang-hackathon:latest`. It's manual +(`workflow_dispatch`), not automatic on every push — trigger it from the +Actions tab whenever the environment needs to pick up new commits, and +definitely once shortly before the event so the pushed image reflects +current `main`. + +This needs GHCR (GitHub Container Registry) enabled for this repository +under the ASF GitHub org. If the workflow's push step fails with a +permissions error, that's an ASF INFRA setting to request, not a bug in the +workflow. + +## Later platforms + +This environment currently exercises Java, Spark, and PostgreSQL, per the +initial hackathon scope. Adding another platform (e.g. SQLite, GraphChi) +only requires seeding/service changes here in `hackathon-CoC2026/` — the +`Dockerfile` and `compose.yaml` at the repository root are untouched. diff --git a/hackathon-CoC2026/compose.yaml b/hackathon-CoC2026/compose.yaml new file mode 100644 index 000000000..6763ce297 --- /dev/null +++ b/hackathon-CoC2026/compose.yaml @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Run from this folder with: docker compose up -d --build +# +# Paths below (context, volumes) are relative to THIS file, so `..` means +# the repository root. + +services: + wayang: + # `docker compose up -d` (no --build) pulls this pre-built image, which + # already has its Maven cache warmed — this is what gets participants + # to a working setup in minutes on hackathon day. `docker compose up -d + # --build` ignores this and builds locally from the Dockerfile instead, + # which is slower (see README.md "Timing") but always matches your + # exact checkout — useful for contributors actively changing the + # Dockerfile/build itself, not needed for hackathon participants. + image: ghcr.io/apache/wayang-hackathon:latest + build: + context: .. + dockerfile: hackathon-CoC2026/Dockerfile + container_name: wayang-hackathon + working_dir: /workspace/wayang + volumes: + # Full repo, read-write, so edits made on the host (or inside the + # container) are shared both ways. + - ..:/workspace/wayang + # Named volume for the Maven local repo: keeps ~/.m2 between + # `docker compose down` / `up` cycles so dependencies aren't + # re-downloaded every time. + - wayang-maven-cache:/root/.m2 + environment: + # Picked up by hackathon-CoC2026/examples/PostgresSmokeTest.java and + # can be used from any app via System.getenv(...) or by writing them + # into a wayang.properties file (see README.md). + WAYANG_POSTGRES_JDBC_URL: jdbc:postgresql://postgres:5432/wayang_hackathon + WAYANG_POSTGRES_USER: wayang + WAYANG_POSTGRES_PASSWORD: wayang + depends_on: + postgres: + condition: service_healthy + stdin_open: true + tty: true + + postgres: + image: postgres:16-alpine + container_name: wayang-hackathon-postgres + environment: + POSTGRES_DB: wayang_hackathon + POSTGRES_USER: wayang + POSTGRES_PASSWORD: wayang + volumes: + - ./docker/postgres/init.sql:/docker-entrypoint-initdb.d/init.sql:ro + - wayang-postgres-data:/var/lib/postgresql/data + ports: + # Exposed on the host too, so participants can point a local SQL + # client at it if they want (e.g. `psql -h localhost -p 55432`). + - "55432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U wayang -d wayang_hackathon"] + interval: 5s + timeout: 5s + retries: 20 + +volumes: + wayang-maven-cache: + wayang-postgres-data: diff --git a/hackathon-CoC2026/docker/entrypoint.sh b/hackathon-CoC2026/docker/entrypoint.sh new file mode 100644 index 000000000..0d1651c4c --- /dev/null +++ b/hackathon-CoC2026/docker/entrypoint.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# The source tree is bind-mounted from the host (see compose.yaml), so mvnw's +# line endings depend on how the host checked it out. Re-normalize on every +# start rather than only at image build time. +if [ -f /workspace/wayang/mvnw ]; then + sed -i 's/\r$//' /workspace/wayang/mvnw + chmod +x /workspace/wayang/mvnw +fi + +exec "$@" diff --git a/hackathon-CoC2026/docker/postgres/init.sql b/hackathon-CoC2026/docker/postgres/init.sql new file mode 100644 index 000000000..9af655462 --- /dev/null +++ b/hackathon-CoC2026/docker/postgres/init.sql @@ -0,0 +1,32 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- Seed table used by examples/PostgresSmokeTest.java to prove the Postgres +-- platform is reachable and queryable from inside the hackathon container. +-- Runs automatically the first time the postgres data volume is created +-- (standard postgres image behavior for /docker-entrypoint-initdb.d). + +CREATE TABLE IF NOT EXISTS word_counts_seed ( + word VARCHAR(64) NOT NULL, + count INTEGER NOT NULL +); + +INSERT INTO word_counts_seed (word, count) VALUES + ('wayang', 42), + ('spark', 17), + ('postgres', 9), + ('hackathon',5), + ('glasgow', 3); diff --git a/hackathon-CoC2026/examples/PostgresSmokeTest.java b/hackathon-CoC2026/examples/PostgresSmokeTest.java new file mode 100644 index 000000000..118a8ca9a --- /dev/null +++ b/hackathon-CoC2026/examples/PostgresSmokeTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.wayang.api.JavaPlanBuilder; +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.WayangContext; +import org.apache.wayang.java.Java; +import org.apache.wayang.postgres.Postgres; +import org.apache.wayang.postgres.operators.PostgresTableSource; +import org.apache.wayang.spark.Spark; + +import java.util.Collection; + +/** + * Hackathon smoke test: builds a single Wayang plan that reads from the + * seeded Postgres table, forces the filter step onto the Java platform, + * and forces the map step onto the Spark platform, proving all three + * platforms work together inside the container. + * + * Run via hackathon-CoC2026/verify.sh, which sets up the classpath needed + * for the Postgres and Spark platforms and compiles/runs this class. + */ +public class PostgresSmokeTest { + + public static void main(String[] args) { + Configuration configuration = new Configuration(); + configuration.setProperty( + "wayang.postgres.jdbc.url", + envOrDefault("WAYANG_POSTGRES_JDBC_URL", "jdbc:postgresql://postgres:5432/wayang_hackathon")); + configuration.setProperty( + "wayang.postgres.jdbc.user", + envOrDefault("WAYANG_POSTGRES_USER", "wayang")); + configuration.setProperty( + "wayang.postgres.jdbc.password", + envOrDefault("WAYANG_POSTGRES_PASSWORD", "wayang")); + + WayangContext wayangContext = new WayangContext(configuration) + .withPlugin(Java.basicPlugin()) + .withPlugin(Spark.basicPlugin()) + .withPlugin(Postgres.plugin()); + + JavaPlanBuilder planBuilder = new JavaPlanBuilder(wayangContext) + .withJobName("Hackathon smoke test (Postgres + Java + Spark)") + .withUdfJarOf(PostgresSmokeTest.class); + + Collection result = planBuilder + .readTable(new PostgresTableSource("word_counts_seed", "word", "count")) + .filter(record -> record.getInt(1) > 5) + .withTargetPlatform(Java.platform()) + .map(record -> record.getString(0).toUpperCase() + " -> " + record.getInt(1)) + .withTargetPlatform(Spark.platform()) + .collect(); + + System.out.println("Wayang plan executed across Postgres, Java, and Spark. Results:"); + result.forEach(System.out::println); + System.out.println("Total rows: " + result.size()); + + if (result.isEmpty()) { + throw new IllegalStateException( + "Connected and ran fine, but got zero rows back — check that init.sql actually ran " + + "(docker compose down -v to reset the postgres volume and retry)."); + } + } + + private static String envOrDefault(String key, String fallback) { + String value = System.getenv(key); + return (value == null || value.isEmpty()) ? fallback : value; + } +} diff --git a/hackathon-CoC2026/verify.sh b/hackathon-CoC2026/verify.sh new file mode 100644 index 000000000..578683206 --- /dev/null +++ b/hackathon-CoC2026/verify.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Run this from inside the wayang dev container: +# docker compose exec wayang bash hackathon-CoC2026/verify.sh +# +# Builds Wayang, extracts the distribution (which has both a jars/ and a +# libs/ directory), resolves the full Maven runtime classpath (needed +# because Spark's own runtime jars are NOT copied into the assembled +# distribution — only wayang-spark's own jar is), and compiles + runs +# PostgresSmokeTest.java against the combined classpath. + +set -euo pipefail +cd "$(dirname "$0")/.." # repo root +REPO_ROOT="$(pwd)" + +DIST_TARBALL="$(find wayang-assembly/target -maxdepth 1 -name 'apache-wayang-assembly-*.tar.gz' | head -n 1)" + +if [ -z "$DIST_TARBALL" ]; then + echo "==> Building wayang-assembly (first run can take significantly longer than" + echo " 15 minutes on a cold Maven cache — this is expected, see README.md)" + ./mvnw -q clean package -pl :wayang-assembly -Pdistribution -DskipTests -B + + DIST_TARBALL="$(find wayang-assembly/target -maxdepth 1 -name 'apache-wayang-assembly-*.tar.gz' | head -n 1)" +fi + +if [ -z "$DIST_TARBALL" ]; then + echo "Could not find apache-wayang-assembly-*.tar.gz under wayang-assembly/target/" >&2 + exit 1 +fi + +echo "==> Extracting $DIST_TARBALL" +tar -xzf "$DIST_TARBALL" -C wayang-assembly/target +DIST_DIR="$(find wayang-assembly/target -maxdepth 1 -type d -name 'wayang-*' | head -n 1)" +export WAYANG_HOME="$(cd "$DIST_DIR" && pwd)" +export PATH="$PATH:$WAYANG_HOME/bin" +echo "WAYANG_HOME=$WAYANG_HOME" +# Confirmed layout: $WAYANG_HOME/{bin,conf,jars,libs} + +echo "==> Verifying Java platform (bundled WordCount app)" +( cd "$WAYANG_HOME" && bin/wayang-submit org.apache.wayang.apps.wordcount.Main java "file://$REPO_ROOT/README.md" | tail -n 20 ) || true +# Uses bin/wayang-submit as-is: it already carries the Java 17 --add-opens/ +# --add-exports flags Spark needs, so this script does not duplicate that +# logic for this check — only for the direct `java` call below, which +# doesn't go through wayang-submit. + +if [ -s /tmp/wayang-classpath.txt ]; then + echo "==> Reusing the Maven classpath baked into this image at build time" +else + echo "==> Resolving the full Maven runtime classpath (covers Spark's transitive" + echo " dependencies, which live in the local Maven repo, not the distribution)" + ./mvnw -q -pl wayang-assembly dependency:build-classpath \ + -Dmdep.outputFile=/tmp/wayang-classpath.txt -B +fi +MAVEN_CP="$(cat /tmp/wayang-classpath.txt)" + +DIST_CP="$WAYANG_HOME/libs/*:$WAYANG_HOME/jars/*" +FULL_CP="${DIST_CP}:${MAVEN_CP}" + +echo "==> Reusing the Java 17 module flags already defined in wayang-submit" +JAVA17_FLAGS="$(grep -o -- '--add-\(opens\|exports\)=[^ "'"'"']*' "$WAYANG_HOME/bin/wayang-submit" | tr '\n' ' ')" + +echo "==> Compiling and running PostgresSmokeTest.java" +mkdir -p /tmp/hackathon-classes +javac -cp "$FULL_CP" -d /tmp/hackathon-classes "$REPO_ROOT/hackathon-CoC2026/examples/PostgresSmokeTest.java" +java $JAVA17_FLAGS -cp "$FULL_CP:/tmp/hackathon-classes" PostgresSmokeTest From 79aa2ca15531a7ccdf5fbf98201aba4d2bd7647f Mon Sep 17 00:00:00 2001 From: Mohit Bisht Date: Tue, 29 Sep 2026 18:22:53 +0530 Subject: [PATCH 03/12] Run hackathon image workflow on branch push --- .github/workflows/hackathon-image.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/hackathon-image.yml b/.github/workflows/hackathon-image.yml index 19cefe437..c18cb1c8b 100644 --- a/.github/workflows/hackathon-image.yml +++ b/.github/workflows/hackathon-image.yml @@ -30,6 +30,9 @@ name: Publish hackathon Docker image on: + push: + branches: + - hackathon-docker-environment workflow_dispatch: {} env: From c5066a7ab7ce3fe6e80a6714e739173b36b8cc95 Mon Sep 17 00:00:00 2001 From: Mohit Bisht Date: Tue, 29 Sep 2026 19:17:56 +0530 Subject: [PATCH 04/12] Use published hackathon image --- hackathon-CoC2026/compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hackathon-CoC2026/compose.yaml b/hackathon-CoC2026/compose.yaml index 6763ce297..ed0c0bc4a 100644 --- a/hackathon-CoC2026/compose.yaml +++ b/hackathon-CoC2026/compose.yaml @@ -28,7 +28,7 @@ services: # which is slower (see README.md "Timing") but always matches your # exact checkout — useful for contributors actively changing the # Dockerfile/build itself, not needed for hackathon participants. - image: ghcr.io/apache/wayang-hackathon:latest + image: ghcr.io/mohit-devlogs/wayang-hackathon:latest build: context: .. dockerfile: hackathon-CoC2026/Dockerfile From eb21f697de8872c373a2eb2cf21e9bc6b17d19aa Mon Sep 17 00:00:00 2001 From: Mohit Bisht Date: Tue, 29 Sep 2026 19:57:11 +0530 Subject: [PATCH 05/12] Remove obsolete Docker setup --- .dockerignore | 18 ++++++++++++++++-- Dockerfile | 34 ---------------------------------- HACKATHON.md | 20 -------------------- compose.yaml | 15 --------------- docker/entrypoint.sh | 9 --------- 5 files changed, 16 insertions(+), 80 deletions(-) delete mode 100644 Dockerfile delete mode 100644 HACKATHON.md delete mode 100644 compose.yaml delete mode 100644 docker/entrypoint.sh diff --git a/.dockerignore b/.dockerignore index 34350adec..adae9ce3f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + .git .gitignore - **/target/ .idea/ @@ -12,4 +26,4 @@ .DS_Store Thumbs.db -.m2/ +.m2/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 95e9718fd..000000000 --- a/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM eclipse-temurin:17-jdk - -WORKDIR /workspace/wayang - -RUN apt-get update \ - && apt-get install -y --no-install-recommends git bash \ - && rm -rf /var/lib/apt/lists/* - -COPY . /workspace/wayang - -RUN sed -i 's/\r$//' mvnw \ - && chmod +x mvnw - -COPY docker/entrypoint.sh /usr/local/bin/wayang-entrypoint - -RUN chmod +x /usr/local/bin/wayang-entrypoint - -ENTRYPOINT ["wayang-entrypoint"] -CMD ["bash"] \ No newline at end of file diff --git a/HACKATHON.md b/HACKATHON.md deleted file mode 100644 index ce6e94045..000000000 --- a/HACKATHON.md +++ /dev/null @@ -1,20 +0,0 @@ -# Wayang Hackathon Development Environment - -This guide provides a Docker-based development environment for the Wayang -hackathon. - -The goal is to allow participants to get a working Wayang environment without -having to install Java, Maven, Spark, Hadoop, or other Wayang dependencies -locally. - -## Prerequisites - -Install: - -- Git -- Docker Desktop - -Verify Docker is available: - -```bash -docker --version \ No newline at end of file diff --git a/compose.yaml b/compose.yaml deleted file mode 100644 index 4f5da9f26..000000000 --- a/compose.yaml +++ /dev/null @@ -1,15 +0,0 @@ -services: - wayang: - build: - context: . - dockerfile: Dockerfile - container_name: wayang-hackathon - working_dir: /workspace/wayang - volumes: - - .:/workspace/wayang - - wayang-maven-cache:/root/.m2 - stdin_open: true - tty: true - -volumes: - wayang-maven-cache: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh deleted file mode 100644 index c4a15bb09..000000000 --- a/docker/entrypoint.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -set -e - -if [ -f /workspace/wayang/mvnw ]; then - sed -i 's/\r$//' /workspace/wayang/mvnw - chmod +x /workspace/wayang/mvnw -fi - -exec "$@" \ No newline at end of file From c23a21154c12a9b12e86c26b2e44598a6a7633c6 Mon Sep 17 00:00:00 2001 From: Mohit Bisht Date: Mon, 5 Oct 2026 00:56:25 +0530 Subject: [PATCH 06/12] Build multi-architecture hackathon image --- .github/workflows/hackathon-image.yml | 79 +++++++++++++++++---------- hackathon-CoC2026/README.md | 30 +++++++--- 2 files changed, 71 insertions(+), 38 deletions(-) diff --git a/.github/workflows/hackathon-image.yml b/.github/workflows/hackathon-image.yml index c18cb1c8b..66401706f 100644 --- a/.github/workflows/hackathon-image.yml +++ b/.github/workflows/hackathon-image.yml @@ -6,33 +6,22 @@ # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. -# Builds hackathon-CoC2026/Dockerfile (which bakes in a warm Maven cache, -# see that file's comments) and publishes it to GitHub Container Registry. -# -# Deliberately manual (workflow_dispatch) rather than on every push: this -# image is for a specific hackathon date, doesn't need to track every -# commit to main, and a full build is expensive. Re-run manually whenever -# the environment needs to pick up new commits before the event. -# -# NOTE: publishing to ghcr.io/apache/... requires GHCR to be enabled for -# this repository under the ASF GitHub org. If this workflow fails on the -# push step with a permissions error, that's an ASF INFRA setting, not a -# problem with this workflow — check with infra or ask in the PR. +# Builds hackathon-CoC2026/Dockerfile for BOTH linux/amd64 and +# linux/arm64, natively on each architecture (no QEMU), and +# publishes a single multi-architecture manifest to GHCR. name: Publish hackathon Docker image on: - push: - branches: - - hackathon-docker-environment workflow_dispatch: {} env: @@ -43,8 +32,21 @@ permissions: packages: write jobs: - build-and-push: - runs-on: ubuntu-latest + build: + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-latest + suffix: amd64 + + - platform: linux/arm64 + runner: ubuntu-24.04-arm + suffix: arm64 + + runs-on: ${{ matrix.runner }} + steps: - name: Check out repository uses: actions/checkout@v4 @@ -59,14 +61,33 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push + - name: Build and push per-architecture image uses: docker/build-push-action@v5 with: context: . file: hackathon-CoC2026/Dockerfile + platforms: ${{ matrix.platform }} push: true - tags: | - ${{ env.IMAGE_NAME }}:latest - ${{ env.IMAGE_NAME }}:${{ github.run_number }} - cache-from: type=gha - cache-to: type=gha,mode=max + tags: ${{ env.IMAGE_NAME }}:latest-${{ matrix.suffix }} + cache-from: type=gha,scope=${{ matrix.suffix }} + cache-to: type=gha,mode=max,scope=${{ matrix.suffix }} + + manifest: + needs: build + runs-on: ubuntu-latest + + steps: + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Create and push multi-arch manifest + run: | + docker buildx imagetools create \ + -t ${{ env.IMAGE_NAME }}:latest \ + -t ${{ env.IMAGE_NAME }}:${{ github.run_number }} \ + ${{ env.IMAGE_NAME }}:latest-amd64 \ + ${{ env.IMAGE_NAME }}:latest-arm64 \ No newline at end of file diff --git a/hackathon-CoC2026/README.md b/hackathon-CoC2026/README.md index 54802500d..ff1c64030 100644 --- a/hackathon-CoC2026/README.md +++ b/hackathon-CoC2026/README.md @@ -14,10 +14,11 @@ beyond Git and Docker Desktop is required on the host. cache takes 30-50+ minutes depending on network speed and machine specs — that's just how long compiling Wayang and its dependency tree (including Spark) genuinely takes the first time. That is NOT what hackathon -participants are expected to do. Instead, the image is built and pushed to -GitHub Container Registry ahead of the event (see `.github/workflows/ -hackathon-image.yml`), with its Maven cache already warmed at build time. -Participants just pull it: +participants are expected to do. Instead, a multi-architecture image +supporting both linux/amd64 and linux/arm64 is built and pushed to +GitHub Container Registry ahead of the event (see +`.github/workflows/hackathon-image.yml`), with its Maven cache +already warmed at build time. Participants just pull the image: ```bash docker compose pull @@ -166,6 +167,14 @@ psql -h postgres -U wayang -d wayang_hackathon -c "select * from word_counts_see confirm `/tmp/wayang-classpath.txt` from `verify.sh` actually contains `spark-sql` and `spark-sql-api` jars; if not, re-run the `dependency: build-classpath` step and check for Maven resolution errors above it. +- **`no matching manifest for linux/arm64/v8`** — the published image does + not contain an ARM64 variant. Re-publish the image using the + multi-architecture workflow above and pull it again with + `docker compose pull`. +- **`tar: ... Function not implemented` on Apple Silicon** — this can occur + when an AMD64-only image is forced to run on an ARM64 host through + emulation. Do not force `linux/amd64`; use the multi-architecture image + so Docker pulls the native ARM64 variant. ## Resetting everything @@ -180,11 +189,14 @@ fresh. ## For maintainers: publishing the image `.github/workflows/hackathon-image.yml` builds `hackathon-CoC2026/Dockerfile` -and pushes it to `ghcr.io/apache/wayang-hackathon:latest`. It's manual -(`workflow_dispatch`), not automatic on every push — trigger it from the -Actions tab whenever the environment needs to pick up new commits, and -definitely once shortly before the event so the pushed image reflects -current `main`. +natively for both `linux/amd64` and `linux/arm64`, publishes the +architecture-specific images, and then creates a single multi-architecture +manifest at `ghcr.io/apache/wayang-hackathon:latest`. + +The workflow is manual (`workflow_dispatch`), not automatic on every push — +trigger it from the Actions tab whenever the environment needs to pick up +new commits, and definitely once shortly before the event so the pushed +image reflects the current environment. This needs GHCR (GitHub Container Registry) enabled for this repository under the ASF GitHub org. If the workflow's push step fails with a From ef6d79430085167772799fe004a8b09ffac2a857 Mon Sep 17 00:00:00 2001 From: Mohit Bisht Date: Mon, 5 Oct 2026 01:11:21 +0530 Subject: [PATCH 07/12] Temporarily trigger hackathon image workflow on branch push --- .github/workflows/hackathon-image.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/hackathon-image.yml b/.github/workflows/hackathon-image.yml index 66401706f..dd21a284b 100644 --- a/.github/workflows/hackathon-image.yml +++ b/.github/workflows/hackathon-image.yml @@ -22,6 +22,9 @@ name: Publish hackathon Docker image on: + push: + branches: + - hackathon-docker-environment workflow_dispatch: {} env: From 935cabad1384e88e8d81536f9f43dd7b5e67d690 Mon Sep 17 00:00:00 2001 From: Mohit Bisht Date: Mon, 5 Oct 2026 01:27:26 +0530 Subject: [PATCH 08/12] Keep hackathon image workflow manual --- .github/workflows/hackathon-image.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/hackathon-image.yml b/.github/workflows/hackathon-image.yml index dd21a284b..66401706f 100644 --- a/.github/workflows/hackathon-image.yml +++ b/.github/workflows/hackathon-image.yml @@ -22,9 +22,6 @@ name: Publish hackathon Docker image on: - push: - branches: - - hackathon-docker-environment workflow_dispatch: {} env: From 825f60fcda599f58433a8d3c5d55228d5d54a233 Mon Sep 17 00:00:00 2001 From: Mohit Bisht Date: Mon, 5 Oct 2026 18:35:19 +0530 Subject: [PATCH 09/12] Fix tar extraction in hackathon container --- hackathon-CoC2026/compose.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hackathon-CoC2026/compose.yaml b/hackathon-CoC2026/compose.yaml index ed0c0bc4a..06fcdb625 100644 --- a/hackathon-CoC2026/compose.yaml +++ b/hackathon-CoC2026/compose.yaml @@ -33,6 +33,8 @@ services: context: .. dockerfile: hackathon-CoC2026/Dockerfile container_name: wayang-hackathon + security_opt: + - seccomp=unconfined working_dir: /workspace/wayang volumes: # Full repo, read-write, so edits made on the host (or inside the From 85b53d8b416b1cfd25387b29ee33d7685a19736f Mon Sep 17 00:00:00 2001 From: Mohit Bisht Date: Sun, 11 Oct 2026 01:37:02 +0530 Subject: [PATCH 10/12] Remove hackathon image publishing workflow --- .github/workflows/hackathon-image.yml | 93 --------------------------- hackathon-CoC2026/README.md | 31 ++++----- 2 files changed, 12 insertions(+), 112 deletions(-) delete mode 100644 .github/workflows/hackathon-image.yml diff --git a/.github/workflows/hackathon-image.yml b/.github/workflows/hackathon-image.yml deleted file mode 100644 index 66401706f..000000000 --- a/.github/workflows/hackathon-image.yml +++ /dev/null @@ -1,93 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Builds hackathon-CoC2026/Dockerfile for BOTH linux/amd64 and -# linux/arm64, natively on each architecture (no QEMU), and -# publishes a single multi-architecture manifest to GHCR. - -name: Publish hackathon Docker image - -on: - workflow_dispatch: {} - -env: - IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/wayang-hackathon - -permissions: - contents: read - packages: write - -jobs: - build: - strategy: - fail-fast: false - matrix: - include: - - platform: linux/amd64 - runner: ubuntu-latest - suffix: amd64 - - - platform: linux/arm64 - runner: ubuntu-24.04-arm - suffix: arm64 - - runs-on: ${{ matrix.runner }} - - steps: - - name: Check out repository - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to GHCR - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push per-architecture image - uses: docker/build-push-action@v5 - with: - context: . - file: hackathon-CoC2026/Dockerfile - platforms: ${{ matrix.platform }} - push: true - tags: ${{ env.IMAGE_NAME }}:latest-${{ matrix.suffix }} - cache-from: type=gha,scope=${{ matrix.suffix }} - cache-to: type=gha,mode=max,scope=${{ matrix.suffix }} - - manifest: - needs: build - runs-on: ubuntu-latest - - steps: - - name: Log in to GHCR - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Create and push multi-arch manifest - run: | - docker buildx imagetools create \ - -t ${{ env.IMAGE_NAME }}:latest \ - -t ${{ env.IMAGE_NAME }}:${{ github.run_number }} \ - ${{ env.IMAGE_NAME }}:latest-amd64 \ - ${{ env.IMAGE_NAME }}:latest-arm64 \ No newline at end of file diff --git a/hackathon-CoC2026/README.md b/hackathon-CoC2026/README.md index ff1c64030..2c080d066 100644 --- a/hackathon-CoC2026/README.md +++ b/hackathon-CoC2026/README.md @@ -16,8 +16,7 @@ that's just how long compiling Wayang and its dependency tree (including Spark) genuinely takes the first time. That is NOT what hackathon participants are expected to do. Instead, a multi-architecture image supporting both linux/amd64 and linux/arm64 is built and pushed to -GitHub Container Registry ahead of the event (see -`.github/workflows/hackathon-image.yml`), with its Maven cache +GitHub Container Registry ahead of the event, with its Maven cache already warmed at build time. Participants just pull the image: ```bash @@ -167,10 +166,10 @@ psql -h postgres -U wayang -d wayang_hackathon -c "select * from word_counts_see confirm `/tmp/wayang-classpath.txt` from `verify.sh` actually contains `spark-sql` and `spark-sql-api` jars; if not, re-run the `dependency: build-classpath` step and check for Maven resolution errors above it. -- **`no matching manifest for linux/arm64/v8`** — the published image does - not contain an ARM64 variant. Re-publish the image using the - multi-architecture workflow above and pull it again with - `docker compose pull`. +- **`no matching manifest for linux/arm64/v8`** — the published image may + not include an ARM64 variant. Ensure the image supports both + `linux/amd64` and `linux/arm64` platforms, then run + `docker compose pull` again. - **`tar: ... Function not implemented` on Apple Silicon** — this can occur when an AMD64-only image is forced to run on an ARM64 host through emulation. Do not force `linux/amd64`; use the multi-architecture image @@ -188,20 +187,14 @@ fresh. ## For maintainers: publishing the image -`.github/workflows/hackathon-image.yml` builds `hackathon-CoC2026/Dockerfile` -natively for both `linux/amd64` and `linux/arm64`, publishes the -architecture-specific images, and then creates a single multi-architecture -manifest at `ghcr.io/apache/wayang-hackathon:latest`. +The hackathon image is published to GitHub Container Registry (GHCR). +To publish an updated image, build and push images for both `linux/amd64` +and `linux/arm64`, then create a multi-architecture manifest for the shared +tag. Ensure the published package is public so participants can pull it +without authenticating. -The workflow is manual (`workflow_dispatch`), not automatic on every push — -trigger it from the Actions tab whenever the environment needs to pick up -new commits, and definitely once shortly before the event so the pushed -image reflects the current environment. - -This needs GHCR (GitHub Container Registry) enabled for this repository -under the ASF GitHub org. If the workflow's push step fails with a -permissions error, that's an ASF INFRA setting to request, not a bug in the -workflow. +After publishing an update, verify that both architectures are present and +that the image passes the end-to-end smoke test. ## Later platforms From 4aca5186574e56b2b6c8858017cdfb0a8f641287 Mon Sep 17 00:00:00 2001 From: Mohit Bisht Date: Sun, 11 Oct 2026 17:14:14 +0530 Subject: [PATCH 11/12] Restore root Dockerfile for CI --- Dockerfile | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..b75a5b82c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM eclipse-temurin:17-jre AS wayang-assembler + +ARG WAYANG_DIST=wayang-assembly/target/*-dist.tar.gz + +COPY ${WAYANG_DIST} /tmp/wayang-dist.tar.gz + +RUN mkdir -p /opt/wayang-root /opt/wayang /opt/wayang/smoke \ + && tar -xzf /tmp/wayang-dist.tar.gz -C /opt/wayang-root \ + && extracted_dir="$(find /opt/wayang-root -mindepth 1 -maxdepth 1 -type d -name 'wayang-*' | head -n 1)" \ + && test -n "${extracted_dir}" \ + && cp -a "${extracted_dir}/." /opt/wayang/ \ + && find /opt/wayang -type f \( -name "*-sources.jar" -o -name "*-javadoc.jar" -o -name "README.md" \) -delete \ + && printf "apache wayang docker smoke test\n" > /opt/wayang/smoke/wordcount.txt \ + && rm /tmp/wayang-dist.tar.gz + +FROM eclipse-temurin:17-jre + +ENV WAYANG_HOME=/opt/wayang +ENV FLAG_WAYANG=true +ENV PATH="${WAYANG_HOME}/bin:${PATH}" + +COPY --from=wayang-assembler /opt/wayang /opt/wayang + +RUN mkdir -p /root/.wayang /opt/wayang/conf \ + && touch /opt/wayang/conf/wayang.properties + +WORKDIR /opt/wayang + +ENTRYPOINT ["/opt/wayang/bin/wayang-submit"] +CMD ["org.apache.wayang.apps.pi.PiEstimation", "java", "1"] From cb22d98c587e6227fece1da1f97b7133ceb36cf3 Mon Sep 17 00:00:00 2001 From: Mohit Bisht Date: Mon, 12 Oct 2026 11:44:24 +0530 Subject: [PATCH 12/12] Fix Docker build context for hackathon environment --- .dockerignore | 35 ++++++++++++++--------- hackathon-CoC2026/Dockerfile.dockerignore | 30 +++++++++++++++++++ 2 files changed, 51 insertions(+), 14 deletions(-) create mode 100644 hackathon-CoC2026/Dockerfile.dockerignore diff --git a/.dockerignore b/.dockerignore index adae9ce3f..185fd7fde 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,17 +13,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -.git -.gitignore -**/target/ - -.idea/ -.vscode/ -*.iml - -*.log - -.DS_Store -Thumbs.db - -.m2/ \ No newline at end of file +* +!Dockerfile +!python/ +python/** +!python/Dockerfile +!python/docker/ +!python/docker/** +!python/examples/ +!python/examples/** +!python/pyproject.toml +!python/setup.cfg +!python/README.md +!python/src/ +python/src/** +!python/src/pywy/ +!python/src/pywy/** +!wayang-assembly/ +wayang-assembly/** +!wayang-assembly/target/ +wayang-assembly/target/** +!wayang-assembly/target/*-dist.tar.gz diff --git a/hackathon-CoC2026/Dockerfile.dockerignore b/hackathon-CoC2026/Dockerfile.dockerignore new file mode 100644 index 000000000..c30f9533f --- /dev/null +++ b/hackathon-CoC2026/Dockerfile.dockerignore @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.git +.gitignore +**/target/ + +.idea/ +.vscode/ +*.iml + +*.log + +.DS_Store +Thumbs.db + +.m2/