Add acceptance tests for the jdbcconfig catalog backend

Add `acceptance-tests-jdbcconfig` make target to test against the jdbcconfig catalog backend. The `acceptance-tests-*` targets change the docker compose project name so the volumes created don't interfere with the ones used during development. Hence new scripts have been added to the `compose/` folder: * `acceptance_datadir` * `acceptance_pgconfig` * `acceptance_jdbcconfig` That allow to run the appropriate `docker compose -f...` and set up the required env variables for each case, like to override the project name and hence the volume names, and to perform manual operations without having to dig up which parameters to use (e.g. `./acceptance_datadir ps`, etc.). The pull request github actions job is modified to upload jar files and download to build the images wihout re-packaging.
2024-10-23 09:24:15 +08:00 · 2024-10-11 19:51:23 -03:00 · 2024-10-11 19:51:23 -03:00 · b4bc6f48d0
commit b4bc6f48d0
parent 41650d9254
8 changed files with 153 additions and 61 deletions
--- a/.github/workflows/pull-request.yaml
+++ b/.github/workflows/pull-request.yaml
@ -15,8 +15,12 @@ on:
      - "acceptance_tests/**"
      - "ci/**"

+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
-  build:
+  test:
    name: Build and Test Pull Request
    if: github.repository == 'geoserver/geoserver-cloud'
    runs-on: ubuntu-latest
@ -26,8 +30,7 @@ jobs:
      uses: actions/checkout@v4
      with:
        submodules: recursive
-
-    - name: Setup Java
+    - name: Set up java
      uses: actions/setup-java@v4
      with:
        distribution: 'temurin'
@ -35,49 +38,70 @@ jobs:
        cache: 'maven'

    - name: Validate source code formatting
-      run: make lint
-
-    - name: Build without tests
      run: |
-        make install
+        make lint

    - name: Test
      run: |
+        # `make test` runs mvn verify, which includes mvn package, so no need to run `package` separatedly for the upload step below
        make test

+    - name: Upload application jar files
+      uses: actions/upload-artifact@v4
+      with:
+        name: application-jars
+        path: ./**/target/*-bin.jar
+        retention-days: 1
+        compression-level: 0 # no compression
+
+  acceptance:
+    name: Acceptance Tests
+    if: github.repository == 'geoserver/geoserver-cloud'
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    needs: test
+    strategy:
+      fail-fast: false
+      matrix:
+        catalog: [ 'datadir', 'pgconfig', 'jdbcconfig' ]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        submodules: recursive
+    - name: Set up java
+      uses: actions/setup-java@v4
+      with:
+        distribution: 'temurin'
+        java-version: '21'
+        cache: 'maven'
+
+    - name: Download application jar files
+      uses: actions/download-artifact@v4
+      with:
+        name: application-jars
+        path: .
+
+    - name: Copy config directory
+      # during the validate phase, the config/ dir is copied to each app's target/config
+      run: ./mvnw validate -ntp -T1C
+
    - name: Build images
-      run: |
-        make build-image
+      # REPACKAGE=false avoids to re-package the apps during build-image
+      run: REPACKAGE=false make build-image

    - name: Install CI dependencies
      run: python3 -m pip install --user --requirement=ci/requirements.txt

-    - name: Run acceptance tests datadir
+    - name: Run acceptance tests ${{ matrix.catalog }}
      run: |
-        make acceptance-tests-datadir
+        make acceptance-tests-${{ matrix.catalog }}

-    - name: Print docker compose logs datadir
-      run: (cd compose && c2cciutils-docker-logs)
+    - name: Print docker compose logs
      if: always()
+      run: (cd compose && c2cciutils-docker-logs)

-    - name: Cleanup acceptance tests datadir
+    - name: Cleanup acceptance tests
+      if: always()
      run: |
-        make clean-acceptance-tests-datadir
-
-    # FIXME: fix pgconfig discrepancies before reactivating
-    # - name: Run acceptance tests pgconfig
-    #   run: |
-    #     make acceptance-tests-pgconfig
-
-    # - name: Print docker compose logs pgconfig
-    #   run: (cd compose && c2cciutils-docker-logs)
-    #   if: always()
-
-    # - name: Cleanup acceptance tests pgconfig
-    #   run: |
-    #     make clean-acceptance-tests-pgconfig
-
-    - name: Remove project jars from cached repository
-      run: |
-        rm -rf ~/.m2/repository/org/geoserver
-        find ~/.m2/repository -name "*SNAPSHOT*" -type d | xargs rm -rf {}
+        make clean-acceptance-tests-${{ matrix.catalog }}
--- a/39
+++ b/39
@ -4,12 +4,6 @@ all: install test build-image
 TAG=$(shell mvn help:evaluate -Dexpression=project.version -q -DforceStdout)

 COSIGN_PASSWORD := $(COSIGN_PASSWORD)
-COMPOSE_PGCONFIG_OPTIONS ?= -f compose.yml -f catalog-pgconfig.yml
-COMPOSE_DATADIR_OPTIONS ?= -f compose.yml -f catalog-datadir.yml
-COMPOSE_ACCEPTANCE_PGCONFIG_OPTIONS ?= --project-name gscloud-acceptance-pgconfig $(COMPOSE_PGCONFIG_OPTIONS) -f acceptance.yml
-COMPOSE_ACCEPTANCE_DATADIR_OPTIONS ?= --project-name gscloud-acceptance-datadir $(COMPOSE_DATADIR_OPTIONS) -f acceptance.yml
-UID=$(shell id -u)
-GID=$(shell id -g)

 REPACKAGE ?= true

@ -19,23 +13,23 @@ clean:

 .PHONY: lint
 lint:
-	./mvnw fmt:check sortpom:verify -Dsort.verifyFailOn=strict -Dsort.verifyFail=stop -ntp
+	./mvnw fmt:check sortpom:verify -Dsort.verifyFailOn=strict -Dsort.verifyFail=stop -ntp -T1C

 .PHONY: format
 format:
-	./mvnw sortpom:sort fmt:format -ntp
+	./mvnw sortpom:sort fmt:format -ntp -T1C

 .PHONY: install
 install:
-	./mvnw clean install -DskipTests -ntp -T4 -U
+	./mvnw clean install -DskipTests -ntp -U -T1C

 .PHONY: package
 package:
-	./mvnw clean package -DskipTests -ntp -T4 -U
+	./mvnw clean package -DskipTests -ntp -U -T1C

 .PHONY: test
 test:
-	./mvnw verify -ntp -T4
+	./mvnw verify -ntp -T1C

 .PHONY: build-image
 build-image: build-base-images build-image-infrastructure build-image-geoserver
@ -135,23 +129,30 @@ build-acceptance:
 	docker build --tag=acceptance:$(TAG) acceptance_tests

 .PHONY: acceptance-tests-pgconfig
-acceptance-tests-pgconfig:
 acceptance-tests-pgconfig: build-acceptance
-	(cd compose/ && TAG=$(TAG) GS_USER=$(UID):$(GID) docker compose $(COMPOSE_ACCEPTANCE_PGCONFIG_OPTIONS) up -d)
-	(cd compose/ && TAG=$(TAG) GS_USER=$(UID):$(GID) docker compose $(COMPOSE_ACCEPTANCE_PGCONFIG_OPTIONS) exec -T acceptance bash -c 'until [ -f /tmp/healthcheck ]; do echo "Waiting for /tmp/healthcheck to be available..."; sleep 5; done && pytest . -vvv --color=yes')
+	(cd compose/ && ./acceptance_pgconfig up -d)
+	(cd compose/ && ./acceptance_pgconfig exec -T acceptance bash -c 'until [ -f /tmp/healthcheck ]; do echo "Waiting for /tmp/healthcheck to be available..."; sleep 5; done && pytest . -vvv --color=yes')

 .PHONY: clean-acceptance-tests-pgconfig
 clean-acceptance-tests-pgconfig:
-	(cd compose/ && TAG=$(TAG) GS_USER=$(UID):$(GID) docker compose $(COMPOSE_ACCEPTANCE_PGCONFIG_OPTIONS) down -v)
+	(cd compose/ && ./acceptance_pgconfig down -v)

 .PHONY: acceptance-tests-datadir
-acceptance-tests-datadir:
 acceptance-tests-datadir: build-acceptance
-	(cd compose/ && TAG=$(TAG) GS_USER=$(UID):$(GID) docker compose $(COMPOSE_ACCEPTANCE_DATADIR_OPTIONS) up -d)
-	(cd compose/ && TAG=$(TAG) GS_USER=$(UID):$(GID) docker compose $(COMPOSE_ACCEPTANCE_DATADIR_OPTIONS) exec -T acceptance bash -c 'until [ -f /tmp/healthcheck ]; do echo "Waiting for /tmp/healthcheck to be available..."; sleep 5; done && pytest . -vvv --color=yes')
+	(cd compose/ && ./acceptance_datadir up -d)
+	(cd compose/ && ./acceptance_datadir exec -T acceptance bash -c 'until [ -f /tmp/healthcheck ]; do echo "Waiting for /tmp/healthcheck to be available..."; sleep 5; done && pytest . -vvv --color=yes')

 .PHONY: clean-acceptance-tests-datadir
 clean-acceptance-tests-datadir:
-	(cd compose/ && TAG=$(TAG) GS_USER=$(UID):$(GID) docker compose $(COMPOSE_ACCEPTANCE_DATADIR_OPTIONS) down -v)
+	(cd compose/ && ./acceptance_datadir down -v)
 	rm -rf compose/catalog-datadir/*
 	touch compose/catalog-datadir/.keep
+
+.PHONY: acceptance-tests-jdbcconfig
+acceptance-tests-jdbcconfig: build-acceptance
+	(cd compose/ && ./acceptance_jdbcconfig up -d)
+	(cd compose/ && ./acceptance_jdbcconfig exec -T acceptance bash -c 'until [ -f /tmp/healthcheck ]; do echo "Waiting for /tmp/healthcheck to be available..."; sleep 5; done && pytest . -vvv --color=yes')
+
+.PHONY: clean-acceptance-tests-jdbcconfig
+clean-acceptance-tests-jdbcconfig:
+	(cd compose/ && ./acceptance_jdbcconfig down -v)
--- a/acceptance_tests/entrypoint.py
+++ b/acceptance_tests/entrypoint.py
@ -29,7 +29,6 @@ def timeout():
 endpoints = [
    f"{GEOSERVER_URL}/wms?SERVICE=WMS&REQUEST=GetCapabilities",
    f"{GEOSERVER_URL}/wfs?SERVICE=WFS&REQUEST=GetCapabilities",
-    f"{GEOSERVER_URL}/wps?SERVICE=WPS&REQUEST=GetCapabilities",
    f"{GEOSERVER_URL}/wcs?SERVICE=WCS&REQUEST=GetCapabilities",
    f"{GEOSERVER_URL}/ows?SERVICE=WMS&REQUEST=GetCapabilities",
    f"{GEOSERVER_URL}/gwc",
--- a/compose/acceptance_datadir
+++ b/compose/acceptance_datadir
@ -0,0 +1,10 @@
+#/bin/bash
+
+UID=$(id -u)
+GID=$(id -g)
+
+GS_USER=$UID:$GID COMPOSE_PROJECT_NAME=gscloud-acceptance-datadir \
+docker compose \
+-f compose.yml \
+-f catalog-datadir.yml \
+-f acceptance.yml "$@"
--- a/compose/acceptance_jdbcconfig
+++ b/compose/acceptance_jdbcconfig
@ -0,0 +1,7 @@
+#/bin/bash
+
+COMPOSE_PROJECT_NAME=gscloud-acceptance-jdbcconfig \
+docker compose \
+-f compose.yml \
+-f catalog-jdbcconfig.yml \
+-f acceptance.yml "$@"
--- a/compose/acceptance_pgconfig
+++ b/compose/acceptance_pgconfig
@ -0,0 +1,7 @@
+#/bin/bash
+
+COMPOSE_PROJECT_NAME=gscloud-acceptance-pgconfig \
+docker compose \
+-f compose.yml \
+-f catalog-pgconfig.yml \
+-f acceptance.yml "$@"
--- a/compose/catalog-jdbcconfig.yml
+++ b/compose/catalog-jdbcconfig.yml
@ -2,6 +2,7 @@ name: gscloud_dev_jdbcconfig

 volumes:
  jdbcconfig_data: # volume for postgresql data, used to store the geoserver config through jdbcconfig
+  datadir:

 #
 # Configures all geoserver services to use the postgresql database server with jdbcconfig as catalog backend.
@ -9,6 +10,14 @@ volumes:
 #

 services:
+  init-sample-data:
+    image: alpine:3.18.4
+    user: ${GS_USER}
+    command: sh -c "cd /opt/app/data_directory; if [ ! -f global.xml ]; then tar xfvz /tmp/datadir.tgz; fi"
+    volumes:
+      - datadir:/opt/app/data_directory
+      - ./catalog-datadir.tgz:/tmp/datadir.tgz
+
  database:
    # be sure geoserver.backend.jdbcconfig.initdb is set to true in application.yml at lease for the first app run
    image: postgres:13-alpine
@ -33,8 +42,13 @@ services:
      JDBCCONFIG_URL: "${JDBCCONFIG_URL}"
      JDBCCONFIG_USERNAME: "${JDBCCONFIG_USERNAME}"
      JDBCCONFIG_PASSWORD: "${JDBCCONFIG_PASSWORD}"
+    volumes:
+      - datadir:/opt/app/data_directory
    depends_on:
-      - database
+      init-sample-data:
+        condition: service_completed_successfully
+      database:
+        condition: service_started

  wms:
    environment:
@ -42,8 +56,13 @@ services:
      JDBCCONFIG_URL: "${JDBCCONFIG_URL}"
      JDBCCONFIG_USERNAME: "${JDBCCONFIG_USERNAME}"
      JDBCCONFIG_PASSWORD: "${JDBCCONFIG_PASSWORD}"
+    volumes:
+      - datadir:/opt/app/data_directory
    depends_on:
-      - database
+      init-sample-data:
+        condition: service_completed_successfully
+      database:
+        condition: service_started

  wcs:
    environment:
@ -51,8 +70,13 @@ services:
      JDBCCONFIG_URL: "${JDBCCONFIG_URL}"
      JDBCCONFIG_USERNAME: "${JDBCCONFIG_USERNAME}"
      JDBCCONFIG_PASSWORD: "${JDBCCONFIG_PASSWORD}"
+    volumes:
+      - datadir:/opt/app/data_directory
    depends_on:
-      - database
+      init-sample-data:
+        condition: service_completed_successfully
+      database:
+        condition: service_started

  rest:
    environment:
@ -60,8 +84,13 @@ services:
      JDBCCONFIG_URL: "${JDBCCONFIG_URL}"
      JDBCCONFIG_USERNAME: "${JDBCCONFIG_USERNAME}"
      JDBCCONFIG_PASSWORD: "${JDBCCONFIG_PASSWORD}"
+    volumes:
+      - datadir:/opt/app/data_directory
    depends_on:
-      - database
+      init-sample-data:
+        condition: service_completed_successfully
+      database:
+        condition: service_started

  webui:
    environment:
@ -69,8 +98,13 @@ services:
      JDBCCONFIG_URL: "${JDBCCONFIG_URL}"
      JDBCCONFIG_USERNAME: "${JDBCCONFIG_USERNAME}"
      JDBCCONFIG_PASSWORD: "${JDBCCONFIG_PASSWORD}"
+    volumes:
+      - datadir:/opt/app/data_directory
    depends_on:
-      - database
+      init-sample-data:
+        condition: service_completed_successfully
+      database:
+        condition: service_started

  gwc:
    environment:
@ -78,5 +112,10 @@ services:
      JDBCCONFIG_URL: "${JDBCCONFIG_URL}"
      JDBCCONFIG_USERNAME: "${JDBCCONFIG_USERNAME}"
      JDBCCONFIG_PASSWORD: "${JDBCCONFIG_PASSWORD}"
+    volumes:
+      - datadir:/opt/app/data_directory
    depends_on:
-      - database
+      init-sample-data:
+        condition: service_completed_successfully
+      database:
+        condition: service_started
--- a/compose/compose.yml
+++ b/compose/compose.yml
@ -125,8 +125,8 @@ services:
    deploy:
      resources:
        limits:
-          cpus: '4.0'
-          memory: 1G
+          cpus: '8.0'
+          memory: 2G

  wfs:
    extends:
@ -172,6 +172,11 @@ services:
    environment:
      JAVA_OPTS: "${JAVA_OPTS_GWC}"
    depends_on: *gs-dependencies
+    deploy:
+      resources:
+        limits:
+          cpus: '8.0'
+          memory: 2G

  rest:
    extends: