Skip to content

Commit 5067f24

Browse files
TheCrab13TheCrab13
committed
test: improve functional tests
1 parent f31ca9d commit 5067f24

12 files changed

Lines changed: 658 additions & 58 deletions

File tree

‎.docker/Dockerfile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
2121
unixodbc-dev \
2222
&& rm -rf /var/lib/apt/lists/*
2323

24+
RUN pip install --no-cache-dir tox
25+
2426
RUN groupadd --gid ${GROUP_ID} dev \
2527
&& useradd --uid ${USER_ID} --gid ${GROUP_ID} --create-home --shell /bin/bash dev
2628

‎.docker/docker-compose.yml‎

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
name: python-db2sql
2+
13
services:
24
dev:
35
build:
@@ -12,6 +14,7 @@ services:
1214
- ..:/workspace
1315
environment:
1416
DEVCONTAINER: "true"
17+
PIPUSER: "0"
1518
HOME: /home/dev
1619
MSSQL_HOST: mssql
1720
MSSQL_PORT: "1433"
@@ -28,7 +31,16 @@ services:
2831
PG_USER: postgres
2932
PG_PASSWORD: ${POSTGRES_PASSWORD:-postgres}
3033
PG_DATABASE: db2sqltarget
34+
PG_SOURCE_SCHEMA: apptest
35+
MYSQL_HOST: mysql
36+
MYSQL_PORT: "3306"
37+
MYSQL_USER: root
38+
MYSQL_PASSWORD: ${MYSQL_ROOT_PASSWORD:-mysqlpw}
39+
MYSQL_DATABASE: db2sqltest
3140
init: true
41+
tty: true
42+
stdin_open: true
43+
command: ["sleep", "infinity"]
3244

3345
mssql:
3446
profiles: ["functional"]
@@ -38,7 +50,7 @@ services:
3850
MSSQL_SA_PASSWORD: ${MSSQL_SA_PASSWORD:-Db2sqlTest!Strong}
3951
MSSQL_PID: Developer
4052
ports:
41-
- "${MSSQL_PORT:-1433}:1433"
53+
- "${MSSQL_PORT:-11433}:1433"
4254
volumes:
4355
- mssql-data:/var/opt/mssql
4456
healthcheck:
@@ -82,7 +94,7 @@ services:
8294
APP_USER: ${ORACLE_APP_USER:-apptest}
8395
APP_USER_PASSWORD: ${ORACLE_APP_PASSWORD:-apptestpw}
8496
ports:
85-
- "${ORACLE_PORT:-1521}:1521"
97+
- "${ORACLE_PORT:-11521}:1521"
8698
volumes:
8799
- ./oracle/init:/container-entrypoint-initdb.d:ro
88100
- oracle-data:/opt/oracle/oradata
@@ -93,6 +105,26 @@ services:
93105
retries: 60
94106
start_period: 60s
95107

108+
mysql:
109+
profiles: ["functional"]
110+
image: mysql:8.4
111+
environment:
112+
MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:-mysqlpw}
113+
MYSQL_DATABASE: db2sqltest
114+
ports:
115+
- "${MYSQL_PORT:-13306}:3306"
116+
volumes:
117+
- ./mysql/init:/docker-entrypoint-initdb.d:ro
118+
- mysql-data:/var/lib/mysql
119+
healthcheck:
120+
test:
121+
- CMD-SHELL
122+
- mysqladmin ping -h 127.0.0.1 -uroot -p"$$MYSQL_ROOT_PASSWORD" --silent
123+
interval: 5s
124+
timeout: 5s
125+
retries: 30
126+
start_period: 20s
127+
96128
postgres:
97129
profiles: ["functional"]
98130
image: postgres:16-alpine
@@ -101,7 +133,7 @@ services:
101133
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres}
102134
POSTGRES_DB: db2sqltarget
103135
ports:
104-
- "${POSTGRES_PORT:-5432}:5432"
136+
- "${PG_PORT:-15432}:5432"
105137
volumes:
106138
- ./postgres/init:/docker-entrypoint-initdb.d:ro
107139
- postgres-data:/var/lib/postgresql/data
@@ -113,5 +145,6 @@ services:
113145

114146
volumes:
115147
mssql-data:
148+
mysql-data:
116149
oracle-data:
117150
postgres-data:

‎.docker/mysql/init/01-schema.sql‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
-- Functional-test fixture for MySQL.
2+
--
3+
-- The MySQLSourceReader uses the connection's database as the "schema"
4+
-- (MySQL has no notion of schema separate from database), so all fixtures
5+
-- live in the ``db2sqltest`` database created from MYSQL_DATABASE.
6+
--
7+
-- Coverage: every MySQL source type referenced by
8+
-- ``PostgresSqlEmitter.DEFAULT_TYPE_MAP`` that is native to MySQL, plus the
9+
-- standard relational fixture (author/book + foreign key + secondary index).
10+
11+
CREATE DATABASE IF NOT EXISTS db2sqltest;
12+
USE db2sqltest;
13+
14+
DROP TABLE IF EXISTS book;
15+
DROP TABLE IF EXISTS author;
16+
DROP TABLE IF EXISTS type_matrix;
17+
18+
-- Type-coverage table -------------------------------------------------------
19+
CREATE TABLE type_matrix (
20+
id INT NOT NULL AUTO_INCREMENT,
21+
c_bit BIT(1) NULL,
22+
c_tinyint TINYINT NULL,
23+
c_smallint SMALLINT NULL,
24+
c_mediumint MEDIUMINT NULL,
25+
c_int INT NULL,
26+
c_bigint BIGINT NULL,
27+
c_decimal DECIMAL(12,4) NULL,
28+
c_numeric NUMERIC(18,6) NULL,
29+
c_float FLOAT NULL,
30+
c_double DOUBLE NULL,
31+
c_char CHAR(8) NULL,
32+
c_varchar VARCHAR(64) NULL,
33+
c_text TEXT NULL,
34+
c_mediumtext MEDIUMTEXT NULL,
35+
c_longtext LONGTEXT NULL,
36+
c_binary BINARY(8) NULL,
37+
c_varbinary VARBINARY(64) NULL,
38+
c_blob BLOB NULL,
39+
c_date DATE NULL,
40+
c_time TIME NULL,
41+
c_datetime DATETIME NULL,
42+
c_timestamp TIMESTAMP NULL DEFAULT NULL,
43+
c_json JSON NULL,
44+
PRIMARY KEY (id)
45+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
46+
47+
-- Relational mini-fixture (parallels mssql/oracle/postgres fixtures)
48+
CREATE TABLE author (
49+
id INT NOT NULL AUTO_INCREMENT,
50+
name VARCHAR(120) NOT NULL,
51+
birth_year INT NULL,
52+
PRIMARY KEY (id)
53+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
54+
55+
CREATE TABLE book (
56+
id INT NOT NULL AUTO_INCREMENT,
57+
author_id INT NOT NULL,
58+
title VARCHAR(200) NOT NULL,
59+
PRIMARY KEY (id),
60+
CONSTRAINT fk_book_author FOREIGN KEY (author_id) REFERENCES author (id)
61+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
62+
63+
CREATE INDEX idx_book_title ON book (title);
64+
65+
-- Representative payload (one populated row + one all-NULL row)
66+
INSERT INTO type_matrix
67+
(c_bit, c_tinyint, c_smallint, c_mediumint, c_int, c_bigint,
68+
c_decimal, c_numeric, c_float, c_double,
69+
c_char, c_varchar, c_text, c_mediumtext, c_longtext,
70+
c_binary, c_varbinary, c_blob,
71+
c_date, c_time, c_datetime, c_timestamp, c_json)
72+
VALUES
73+
(b'1', 7, 32100, 8388600, 2147483640, 9000000000000000000,
74+
1234.5678, 1234567.890123, 1.5, 3.141592653589793,
75+
'fixed', 'with accent é', 'long text payload', 'medium text', 'long text',
76+
UNHEX('DEADBEEFCAFEBABE'), UNHEX('01020304'), UNHEX('05060708'),
77+
'2024-01-31', '14:30:00', '2024-01-31 14:30:00',
78+
'2024-01-31 14:30:00', JSON_OBJECT('k', 'v')),
79+
(NULL, NULL, NULL, NULL, NULL, NULL,
80+
NULL, NULL, NULL, NULL,
81+
NULL, NULL, NULL, NULL, NULL,
82+
NULL, NULL, NULL,
83+
NULL, NULL, NULL, NULL, NULL);
84+
85+
INSERT INTO author (name, birth_year) VALUES ('Alice', 1980);
86+
INSERT INTO author (name, birth_year) VALUES ('Bob', NULL);
87+
88+
INSERT INTO book (author_id, title) VALUES (1, 'First');
89+
INSERT INTO book (author_id, title) VALUES (1, 'Second\'s ride');
90+
INSERT INTO book (author_id, title) VALUES (2, 'Bob book');

‎.docker/postgres/init/01-schema.sql‎

Lines changed: 91 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,93 @@
1-
-- Target PostgreSQL database used by functional tests to validate a generated
2-
-- dump. The dump is supposed to be self-sufficient (CREATE SCHEMA + DDL +
3-
-- COPY/INSERT), so this init script only ensures the database exists and is
4-
-- otherwise empty.
1+
-- Functional-test fixture for PostgreSQL.
2+
--
3+
-- The "db2sqltarget" database (created from POSTGRES_DB by the entrypoint) is
4+
-- used both as:
5+
-- * a TARGET database for dumps produced by db2sql (apply step) — left
6+
-- untouched outside the apptest schema below;
7+
-- * a SOURCE database whose ``apptest`` schema mirrors the mssql/oracle
8+
-- fixtures (type_matrix + author/book) so the PostgreSQL reader can be
9+
-- exercised by the functional suite.
10+
--
11+
-- The reader filters out pg_catalog/information_schema/pg_toast, so the
12+
-- ``apptest`` schema is the only one surfaced by collect_metadata().
13+
-- Note: types covered are the PG-native ones referenced by
14+
-- ``PostgresSqlEmitter.DEFAULT_TYPE_MAP`` (no MSSQL/Oracle-only types).
515

6-
-- The database "db2sqltarget" is created from POSTGRES_DB by the entrypoint,
7-
-- so nothing else is strictly required here. We keep the file so the volume
8-
-- mount /docker-entrypoint-initdb.d is non-empty and to host any future
9-
-- preconditioning (extensions, roles, etc.).
16+
CREATE SCHEMA IF NOT EXISTS apptest;
1017

11-
SELECT 'postgres target ready' AS status;
18+
DROP TABLE IF EXISTS apptest.book;
19+
DROP TABLE IF EXISTS apptest.author;
20+
DROP TABLE IF EXISTS apptest.type_matrix;
21+
22+
-- Type-coverage table -------------------------------------------------------
23+
CREATE TABLE apptest.type_matrix (
24+
id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
25+
c_boolean BOOLEAN,
26+
c_smallint SMALLINT,
27+
c_integer INTEGER,
28+
c_bigint BIGINT,
29+
c_real REAL,
30+
c_double DOUBLE PRECISION,
31+
c_numeric NUMERIC(18, 6),
32+
c_decimal DECIMAL(12, 4),
33+
c_char CHAR(8),
34+
c_varchar VARCHAR(64),
35+
c_text TEXT,
36+
c_bytea BYTEA,
37+
c_date DATE,
38+
c_time TIME,
39+
c_timestamp TIMESTAMP,
40+
c_timestamptz TIMESTAMP WITH TIME ZONE,
41+
c_uuid UUID,
42+
c_json JSON,
43+
c_jsonb JSONB,
44+
c_xml XML
45+
);
46+
47+
-- Relational mini-fixture (parallels mssql/oracle/sqlite fixtures)
48+
CREATE TABLE apptest.author (
49+
id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
50+
name VARCHAR(120) NOT NULL,
51+
birth_year INTEGER
52+
);
53+
54+
CREATE TABLE apptest.book (
55+
id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
56+
author_id INTEGER NOT NULL,
57+
title VARCHAR(200) NOT NULL,
58+
CONSTRAINT fk_book_author FOREIGN KEY (author_id) REFERENCES apptest.author (id)
59+
);
60+
61+
CREATE INDEX idx_book_title ON apptest.book (title);
62+
63+
-- Representative payload (one populated row + one all-NULL row)
64+
INSERT INTO apptest.type_matrix
65+
(c_boolean, c_smallint, c_integer, c_bigint,
66+
c_real, c_double, c_numeric, c_decimal,
67+
c_char, c_varchar, c_text, c_bytea,
68+
c_date, c_time, c_timestamp, c_timestamptz,
69+
c_uuid, c_json, c_jsonb, c_xml)
70+
VALUES
71+
(TRUE, 32100, 2147483640, 9000000000000000000,
72+
1.5, 3.141592653589793, 1234567.890123, 1234.5678,
73+
'fixed', 'with accent é', 'long text payload', '\xDEADBEEFCAFEBABE',
74+
DATE '2024-01-31', TIME '14:30:00',
75+
TIMESTAMP '2024-01-31 14:30:00',
76+
TIMESTAMP WITH TIME ZONE '2024-01-31 14:30:00+01',
77+
'11111111-2222-3333-4444-555555555555',
78+
'{"k": "v"}', '{"k": "v"}',
79+
XMLPARSE(DOCUMENT '<root><k>v</k></root>')),
80+
(NULL, NULL, NULL, NULL,
81+
NULL, NULL, NULL, NULL,
82+
NULL, NULL, NULL, NULL,
83+
NULL, NULL, NULL, NULL,
84+
NULL, NULL, NULL, NULL);
85+
86+
INSERT INTO apptest.author (name, birth_year) VALUES ('Alice', 1980);
87+
INSERT INTO apptest.author (name, birth_year) VALUES ('Bob', NULL);
88+
89+
INSERT INTO apptest.book (author_id, title) VALUES (1, 'First');
90+
INSERT INTO apptest.book (author_id, title) VALUES (1, 'Second''s ride');
91+
INSERT INTO apptest.book (author_id, title) VALUES (2, 'Bob book');
92+
93+
SELECT 'postgres apptest fixture loaded' AS status;

‎CONTRIBUTING.rst‎

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,7 @@ You need to install ``tox`` using one of the following approach:
229229
Running integration tests
230230
-------------------------
231231

232-
Integration tests run against a local symbol store.
232+
Integration tests run against a local symbol store.
233233

234234
To run these tests:
235235

@@ -239,6 +239,51 @@ To run these tests:
239239
tox -e cli
240240
241241
242+
Running functional tests
243+
------------------------
244+
245+
Functional tests exercise the source readers (and emitters) against real
246+
database servers. A docker compose stack under ``.docker/`` brings up the
247+
required services (MSSQL, MySQL, PostgreSQL, and optionally Oracle) and
248+
applies the per-DB init scripts in ``.docker/<db>/init/``.
249+
250+
The ``functional`` docker compose profile covers MSSQL, MySQL and Postgres
251+
(lightweight images). Oracle lives in its own ``oracle`` profile because the
252+
image is ~10 GB and takes 3–5 min to boot.
253+
254+
Make targets:
255+
256+
.. code-block:: bash
257+
258+
# full stack (mssql + mysql + postgres + oracle), then run all functional
259+
# tests inside the dev container
260+
make stack-up
261+
make test-functional
262+
263+
# lightweight subset — no Oracle (fast iteration)
264+
make stack-up-light
265+
make test-functional-light
266+
267+
# Oracle-only (heavy, on-demand)
268+
make stack-up-oracle
269+
make test-oracle
270+
271+
# tear down (keep volumes / wipe volumes)
272+
make stack-down
273+
make stack-reset
274+
275+
The connection parameters consumed by the test fixtures are injected into
276+
the dev container by ``.docker/docker-compose.yml`` (``MSSQL_*``, ``MYSQL_*``,
277+
``PG_*``, ``ORACLE_*``). When a server is unreachable or its env vars are
278+
missing, the corresponding fixture skips the test with a clear message
279+
instead of failing — so the suite stays green when run outside the stack.
280+
281+
Adding a new functional fixture: drop an init script under
282+
``.docker/<db>/init/`` (executed automatically by the official images) and
283+
mirror the existing tests in ``tests/functional/`` (one test module per
284+
source DB, ``pytestmark = pytest.mark.functional``).
285+
286+
242287
Releases
243288
--------
244289

‎Makefile‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -44,14 +44,16 @@ docker-clean: ## Remove the docker dev image and associated resources
4444
# `oracle` profile because the image is ~10 GB and takes 3-5 min to boot.
4545
# `stack-up` keeps the historical "everything" behaviour for local dev.
4646

47-
stack-up: ## Start the full functional DB stack (mssql, postgres, oracle) and wait until healthy
48-
$(DOCKER_ENV) $(DOCKER_COMPOSE) --profile functional --profile oracle up -d --wait
47+
stack-up: ## Start the full functional DB stack (mssql, mysql, postgres, oracle) and wait until healthy
48+
$(DOCKER_ENV) $(DOCKER_COMPOSE) --profile functional --profile oracle up -d --wait mssql mysql postgres oracle
49+
$(DOCKER_ENV) $(DOCKER_COMPOSE) --profile functional run --rm mssql-init
4950

50-
stack-up-light: ## Start only the lightweight functional DB stack (mssql, postgres) — no Oracle
51-
$(DOCKER_ENV) $(DOCKER_COMPOSE) --profile functional up -d --wait
51+
stack-up-light: ## Start only the lightweight functional DB stack (mssql, mysql, postgres) — no Oracle
52+
$(DOCKER_ENV) $(DOCKER_COMPOSE) --profile functional up -d --wait mssql mysql postgres
53+
$(DOCKER_ENV) $(DOCKER_COMPOSE) --profile functional run --rm mssql-init
5254

5355
stack-up-oracle: ## Start only the Oracle container (heavy, ~3-5 min boot) — no mssql/postgres
54-
$(DOCKER_ENV) $(DOCKER_COMPOSE) --profile oracle up -d --wait
56+
$(DOCKER_ENV) $(DOCKER_COMPOSE) --profile oracle up -d --wait oracle
5557

5658
stack-down: ## Stop the functional DB stack (keep volumes)
5759
$(DOCKER_COMPOSE) --profile functional --profile oracle down

0 commit comments

Comments
 (0)