Index: trunk/.dockerignore =================================================================== --- trunk/.dockerignore (revision 959) +++ trunk/.dockerignore (revision 959) @@ -0,0 +1,3 @@ +/Dockerfile +/docker-compose.yml +/docker/ Index: trunk/Dockerfile =================================================================== --- trunk/Dockerfile (revision 959) +++ trunk/Dockerfile (revision 959) @@ -0,0 +1,164 @@ +# +# Base: Ubuntu 18.04 with updates and external packages +# +FROM ubuntu:bionic-20200713 AS base +ARG DEBIAN_FRONTEND=noninteractive +ARG BUILD_DEPS=" \ + dirmngr \ + gpg-agent \ + software-properties-common \ +" +ARG RUN_DEPS=" \ + libcurl3-gnutls \ + libfcgi \ + libmapserver-dev \ + libmozjs185-dev \ + libpq5 \ + libpython2.7 \ + libxslt1.1 \ + gdal-bin \ + python \ + r-base \ +" +RUN set -ex \ + && apt-get update && apt-get install -y --no-install-recommends $BUILD_DEPS \ + \ + && add-apt-repository ppa:osgeolive/nightly \ + && add-apt-repository ppa:ubuntugis/ppa \ + && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 \ + && add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' \ + \ + && apt-get install -y $RUN_DEPS \ + \ + && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false $BUILD_DEPS \ + && rm -rf /var/lib/apt/lists/* + +# +# builder1: base image with zoo-kernel +# +FROM base AS builder1 +ARG DEBIAN_FRONTEND=noninteractive +ARG BUILD_DEPS=" \ + bison \ + flex \ + make \ + autoconf \ + gcc \ + gettext-base \ + \ + libfcgi-dev \ + libgdal-dev \ + libjson-c-dev \ + libssh2-1-dev \ + libssl-dev \ + libxml2-dev \ + libxslt1-dev \ + python-dev \ + uuid-dev \ + r-base-dev \ +" +WORKDIR /zoo-project +COPY . . + +RUN set -ex \ + && apt-get update && apt-get install -y --no-install-recommends $BUILD_DEPS \ + \ + && make -C ./thirds/cgic206 libcgic.a \ + \ + && cd ./zoo-project/zoo-kernel \ + && autoconf \ + && ./configure --with-python --with-js=/usr --with-mapserver=/usr --with-ms-version=7 --with-json=/usr --with-db-backend --prefix=/usr \ + && make \ + && make install \ + \ + # TODO: why not copied by 'make'? + && cp zoo_loader.cgi main.cfg /usr/lib/cgi-bin/ \ + && cp ../zoo-services/hello-py/cgi-env/* /usr/lib/cgi-bin/ \ + && cp ../zoo-services/hello-js/cgi-env/* /usr/lib/cgi-bin/ \ + && cp ../zoo-api/js/* /usr/lib/cgi-bin/ \ + && cp ../zoo-services/hello-py/cgi-env/* /usr/lib/cgi-bin/ \ + && cp ../zoo-services/hello-js/cgi-env/* /usr/lib/cgi-bin/ \ + && cp ../zoo-api/js/* /usr/lib/cgi-bin/ \ + \ + && cp oas.cfg /usr/lib/cgi-bin/ \ + \ + # TODO: main.cfg is not processed \ + && prefix=/usr envsubst < main.cfg > /usr/lib/cgi-bin/main.cfg \ + \ + && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false $BUILD_DEPS \ + && rm -rf /var/lib/apt/lists/* + +# +# Optional zoo modules build. +# +FROM base AS builder2 +ARG DEBIAN_FRONTEND=noninteractive +ARG BUILD_DEPS=" \ + bison \ + flex \ + make \ + autoconf \ + g++ \ + gcc \ + libc-dev \ + libfcgi-dev \ + libgdal-dev \ + libxml2-dev \ + libxslt1-dev \ +" +WORKDIR /zoo-project +COPY ./zoo-project/zoo-services/utils/status ./zoo-project/zoo-services/utils/status + +# From zoo-kernel +COPY --from=builder1 /usr/lib/cgi-bin/ /usr/lib/cgi-bin/ +COPY --from=builder1 /usr/lib/libzoo_service.so.1.5 /usr/lib/libzoo_service.so.1.5 +COPY --from=builder1 /usr/lib/libzoo_service.so /usr/lib/libzoo_service.so +COPY --from=builder1 /usr/com/zoo-project/ /usr/com/zoo-project/ +COPY --from=builder1 /usr/include/zoo/ /usr/include/zoo/ + +# Additional files from bulder2 +COPY --from=builder1 /zoo-project/zoo-project/zoo-kernel/ZOOMakefile.opts /zoo-project/zoo-project/zoo-kernel/ZOOMakefile.opts +COPY --from=builder1 /zoo-project/zoo-project/zoo-kernel/sqlapi.h /zoo-project/zoo-project/zoo-kernel/sqlapi.h +COPY --from=builder1 /zoo-project/zoo-project/zoo-kernel/service.h /zoo-project/zoo-project/zoo-kernel/service.h +COPY --from=builder1 /zoo-project/zoo-project/zoo-kernel/service_internal.h /zoo-project/zoo-project/zoo-kernel/service_internal.h +COPY --from=builder1 /zoo-project/zoo-project/zoo-kernel/version.h /zoo-project/zoo-project/zoo-kernel/version.h + +RUN set -ex \ + && apt-get update && apt-get install -y --no-install-recommends $BUILD_DEPS \ + \ + && cd ./zoo-project/zoo-services/utils/status \ + && make \ + && make install \ + \ + && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false $BUILD_DEPS \ + && rm -rf /var/lib/apt/lists/* + +# +# Runtime image with apache2. +# +FROM base AS runtime +ARG DEBIAN_FRONTEND=noninteractive +ARG RUN_DEPS=" \ + apache2 \ + curl \ +" + +# From zoo-kernel +COPY --from=builder1 /usr/lib/cgi-bin/ /usr/lib/cgi-bin/ +COPY --from=builder1 /usr/lib/libzoo_service.so.1.5 /usr/lib/libzoo_service.so.1.5 +COPY --from=builder1 /usr/lib/libzoo_service.so /usr/lib/libzoo_service.so +COPY --from=builder1 /usr/com/zoo-project/ /usr/com/zoo-project/ +COPY --from=builder1 /usr/include/zoo/ /usr/include/zoo/ + +# From optional zoo modules +COPY --from=builder2 /usr/lib/cgi-bin/ /usr/lib/cgi-bin/ +COPY --from=builder2 /usr/com/zoo-project/ /usr/com/zoo-project/ + +RUN set -ex \ + && apt-get update && apt-get install -y --no-install-recommends $RUN_DEPS \ + \ + && rm -rf /var/lib/apt/lists/* \ + && a2enmod cgi + +EXPOSE 80 +CMD /usr/sbin/apache2ctl -D FOREGROUND Index: trunk/docker-compose.yml =================================================================== --- trunk/docker-compose.yml (revision 959) +++ trunk/docker-compose.yml (revision 959) @@ -0,0 +1,35 @@ +version: '3' +services: + zookernel: + build: . + ports: + - "80:80" + volumes: + - ./docker/main.cfg:/usr/lib/cgi-bin/main.cfg + - ./docker/oas.cfg:/usr/lib/cgi-bin/oas.cfg + - ./docker/default.conf:/etc/apache2/sites-available/000-default.conf + depends_on: + - pgbouncer + pg: + image: postgres:9.6.18-alpine + restart: always + environment: + POSTGRES_USER: zoo + POSTGRES_PASSWORD: zoo + POSTGRES_DB: zoo + volumes: + # Load the zoo kernel schema + - ./zoo-project/zoo-kernel/sql/schema.sql:/docker-entrypoint-initdb.d/1-schema.sql + pgbouncer: + image: edoburu/pgbouncer:1.8.1 + environment: + DATABASE_URL: "postgres://zoo:zoo@pg/zoo" + MAX_CLIENT_CONN: 1000 + MAX_DB_CONNECTIONS: 100 + DEFAULT_POOL_SIZE: 100 + POOL_MODE: transaction + ports: + - "5432:5432" + depends_on: + - pg + Index: trunk/docker/default.conf =================================================================== --- trunk/docker/default.conf (revision 959) +++ trunk/docker/default.conf (revision 959) @@ -0,0 +1,31 @@ + + # The ServerName directive sets the request scheme, hostname and port that + # the server uses to identify itself. This is used when creating + # redirection URLs. In the context of virtual hosts, the ServerName + # specifies what hostname must appear in the request's Host: header to + # match this virtual host. For the default virtual host (this file) this + # value is not decisive as it is used as a last resort host regardless. + # However, you must set it for any further virtual host explicitly. + #ServerName www.example.com + + ServerAdmin webmaster@localhost + DocumentRoot /var/www/html + + # Available loglevels: trace8, ..., trace1, debug, info, notice, warn, + # error, crit, alert, emerg. + # It is also possible to configure the loglevel for particular + # modules, e.g. + #LogLevel info ssl:warn + + #ErrorLog ${APACHE_LOG_DIR}/error.log + #CustomLog ${APACHE_LOG_DIR}/access.log combined + ErrorLog /dev/stderr + TransferLog /dev/stdout + + # For most configuration files from conf-available/, which are + # enabled or disabled at a global level, it is possible to + # include a line for only one particular virtual host. For example the + # following line enables the CGI configuration for this host only + # after it has been globally disabled with "a2disconf". + #Include conf-available/serve-cgi-bin.conf + Index: trunk/docker/main.cfg =================================================================== --- trunk/docker/main.cfg (revision 959) +++ trunk/docker/main.cfg (revision 959) @@ -0,0 +1,42 @@ +[main] +encoding = utf-8 +version = 1.0.0 +serverAddress = http://www.zoo-project.org/zoo/ +language = en-US +lang = fr-FR,en-CA,en-US +tmpPath=/tmp/ +tmpUrl = ../tmpPathRelativeToServerAdress/ +dataPath = /usr/com/zoo-project +cacheDir = /tmp/ + +[identification] +title = The ZOO-Project OGC WPS Development Server +abstract = Development version of ZOO-Project OGC WPS. See http://www.zoo-project.org +fees = None +accessConstraints = none +keywords = WPS,GIS,buffer + +[provider] +providerName=ZOO-Project +providerSite=http://www.zoo-project.org +individualName=Gerald FENOY +positionName=Developer +role=Dev +addressDeliveryPoint=1280, avenue des Platanes +addressCity=Lattes +addressAdministrativeArea=False +addressPostalCode=34970 +addressCountry=fr +addressElectronicMailAddress=gerald.fenoy@geolabs.fr +phoneVoice=False +phoneFacsimile=False + +[database] +dbname=zoo +port=5432 +user=zoo +password=zoo +host=pgbouncer +type=PG +schema=public + Index: trunk/docker/oas.cfg =================================================================== --- trunk/docker/oas.cfg (revision 959) +++ trunk/docker/oas.cfg (revision 959) @@ -0,0 +1,197 @@ +[openapi] +rootUrl=http://localhost/cgi-bin/zoo_loader.cgi +links=/,/api,/conformance,/processes +paths=/root,/api,/conformance,/processes,/processes/{id},/processes/{id}/jobs,/processes/{id}/jobs/{jobID},/processes/{id}/jobs/{jobID}/result +parameters=id,jobID,resultID +header_parameters=oas-header1,oas-header2 +version=3.0.1 + +[oas-header1] +in=header +name=Prefer +type=string +required=false +enum=respond-async + +[oas-header2] +in=header +name=Prefer +type=string +required=false +enum=return=representation,return=minimal + +[/] +rel=self +type=application/json +title=this document + +[root] +method=get +title=landing page of this API +abstract=The landing page provides links to the API definition, the Conformance statements and the metadata about the processes offered by this service. +tags=Home +tags_description= +schema=https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/schemas/root.yaml + +[/api] +rel=service +type=application/openapi+json;version=3.0 +title=the API definition + +[api] +method=get +title=This document +abstract=This document +tags=API +tags_description= +schema= + +[/conformance] +rel=conformance +type=application/json +title=WPS 2.0 REST/JSON Binding Extension conformance classes implemented by this server + +[conformance] +method=get +title=information about standards that this API conforms to +abstract==list all requirements classes specified in a standard (e.g., WPS REST/JSON Binding Core) that the server conforms to +tags=Conformance +tags_description= +schema=https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/schemas/req-classes.yaml + +[/processes] +rel=processes +type=application/json +title=The processes offered by this server + +[processes] +method=get +title=retrieve available processes +abstract=Information about the available processes +tags=GetCapabilities +tags_description= +schema=https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/schemas/processCollection.yaml + +[processes/{id}] +method=get +title=retrieve a process description +abstract=A process description. +tags=ProcessDescription +tags_description= +schema=https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/schemas/process.yaml +parameters=/components/parameters/id + + +[/processes/{id}/jobs] +rel=canonical +type=application/json +title=Process Description + +[processes/{id}/jobs] +length=2 +method=get +title=retrieve a list of jobs run +abstract=A list of jobs run. +tags=JobList +tags_description= +schema=https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/schemas/jobCollection.yaml +parameters=/components/parameters/id +method_1=post +title_1=execute a job +abstract_1=An execute endpoint. +tags_1=Execute Endpoint +tags_description_1= +schema_1=https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/schemas/result.yaml +parameters_1=/components/parameters/id,/components/parameters/oas-header1 + +[requestBody] +abstract=Mandatory execute request in JSON format +type=application/json +schema=https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/schemas/execute.yaml + +[/processes/{id}/jobs/{jobID}] +rel=canonical +type=application/json +title=Status + +[processes/{id}/jobs/{jobID}] +method=get +title=The status of a job. +abstract=The status of a job. +tags=GetStatus +tags_description= +schema=https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/schemas/statusInfo.yaml +parameters=/components/parameters/id,/components/parameters/jobID + +[/processes/{id}/jobs/{jobID}/result] +rel=canonical +type=application/json +title=Get Result + +[processes/{id}/jobs/{jobID}/result] +method=get +title=The result of a job execution. +abstract=The result of a job execution. +tags=GetResult +tags_description= +schema=https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/schemas/result.yaml +parameters=/components/parameters/id,/components/parameters/jobID +ep=,/components/parameters/oas-header2 + +[/processes/{id}/jobs/{jobID}/result/{resultID}] +rel=canonical +type=application/json +title=Get Result + +[processes/{id}/jobs/{jobID}/result/{resultID}] +method=get +title=The result of a job execution. +abstract=The result of a job execution. +tags=GetResult +tags_description= +schema=https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/schemas/valueType.yaml +parameters=/components/parameters/id,/components/parameters/jobID,/components/parameters/resultID,/components/parameters/oas-header2 + + +[{id}] +type=string +title=The id of a process +abstract=The id of a process +in=path +required=true +example=buffer + +[{jobID}] +type=string +title=The id of a job +abstract=The id of a job +in=path +required=true + +[{resultID}] +type=string +title=The id of an output +abstract=The id of an output +in=path +required=true + +[{f}] +default=json +enum=json +title=The optional f parameter +abstract=The optional f parameter indicates the output format which the server shall provide as part of the response document. The default format is JSON. +in=query +required=false + +[conformTo] +rootUrl=http://www.opengis.net/spec/WPS/2.0/req/service/binding/rest-json/ +link=core +link_1=oas30 +link_2=json +length=3 + +[exception] +abstract=Exception +type=application/json +schema=https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/schemas/exception.yaml + Index: trunk/docker/test_long_process.py =================================================================== --- trunk/docker/test_long_process.py (revision 959) +++ trunk/docker/test_long_process.py (revision 959) @@ -0,0 +1,94 @@ +import argparse +import requests +import time + +from threading import Thread +from xml.etree import ElementTree + +parser = argparse.ArgumentParser() +parser.add_argument("threads", help="number of threads", type=int) +args = parser.parse_args() + + +class LongProcess(Thread): + def __init__(self, name): + Thread.__init__(self) + self.name = name + self.progress = None + self.location_url = self.launch_long_process() + print("INIT %s %s" % (self.name, self.location_url)) + + def launch_long_process(self): + endpoint_url = "http://localhost/cgi-bin/zoo_loader.cgi" + params = dict( + request="Execute", + service="WPS", + version="1.0.0", + Identifier="longProcess", + DataInputs="a=toto", + ResponseDocument="Result", + storeExecuteResponse="true", + status="true", + ) + r = requests.get(endpoint_url, params=params) + root = ElementTree.fromstring(r.content) + + location_url = root.attrib["statusLocation"] + return location_url + + def get_progress(self): + progress = None + r = requests.get(self.location_url) + root = ElementTree.fromstring(r.content) + ns = dict(wps="http://www.opengis.net/wps/1.0.0") + status = root.find("wps:Status", ns) + if status: + ctime = status.attrib["creationTime"] + started = root.find("wps:Status/wps:ProcessStarted", ns) + #started = root.find("wps:ProcessStarted", ns) + succeeded = root.find("wps:Status/wps:ProcessSucceeded", ns) + #succeeded = root.find("wps:ProcessSucceeded", ns) + if started is not None and started.attrib["percentCompleted"]: + percent = started.attrib["percentCompleted"] + progress = int(percent) + elif succeeded is not None: + progress = 100 + else: + # error? + print(r.content.decode()) + progress = None + else: + print(r.content.decode()) + progress = -1 + + self.progress = progress + return self.progress + + def run(self): + print("Thread %s: starting" % self.name) + while True: + progress = self.get_progress() + if progress == 100: + print("Thread %s: succeeded" % self.name) + break + elif progress == -1: + print("Thread %s: failed" % self.name) + break + elif progress is None: + print("Thread %s: error" % self.name) + break + else: + print("Thread %s :progress %s" % (self.name, progress)) + time.sleep(1.0) + print("Thread %s: finishing" % self.name) + + +threads = list() +for i in range(0, args.threads): + t = LongProcess(i) + t.start() + threads.append(t) + +for t in threads: + t.join() + Index: trunk/zoo-project/zoo-kernel/sql/schema.sql =================================================================== --- trunk/zoo-project/zoo-kernel/sql/schema.sql (revision 958) +++ trunk/zoo-project/zoo-kernel/sql/schema.sql (revision 959) @@ -37,6 +37,6 @@ -- Used to store information about services running asynchronously create table services ( - osid TEXT unique, - sid TEXT unique, + osid TEXT, + sid TEXT, uuid TEXT unique, fstate varchar(25),