Index: branches/prototype-v0/zoo-project/zoo-kernel/Makefile.in =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/Makefile.in (revision 851) +++ branches/prototype-v0/zoo-project/zoo-kernel/Makefile.in (revision 854) @@ -45,5 +45,5 @@ service_callback.o: service_callback.c - g++ -fPIC ${JSON_CFLAGS} ${CFLAGS} -c service_callback.c + g++ -fPIC ${XML2CFLAGS} ${JSON_CFLAGS} ${CFLAGS} -c service_callback.c request_parser.o: request_parser.c request_parser.h @@ -60,8 +60,8 @@ server_internal.o: server_internal.c server_internal.h service.h mimetypes.h - g++ ${GDAL_CFLAGS} ${JS_ENABLED} ${JSCFLAGS} ${XML2CFLAGS} ${CFLAGS} -fPIC -c server_internal.c + g++ ${JSON_CFLAGS} ${GDAL_CFLAGS} ${JS_ENABLED} ${JSCFLAGS} ${XML2CFLAGS} ${CFLAGS} -fPIC -c server_internal.c service_internal.o: service_internal.c service_internal.h service.h - gcc ${GDAL_CFLAGS} ${JS_ENABLED} ${JSCFLAGS} ${XML2CFLAGS} ${CFLAGS} -fPIC -c service_internal.c + gcc ${JSON_CFLAGS} ${GDAL_CFLAGS} ${JS_ENABLED} ${JSCFLAGS} ${XML2CFLAGS} ${CFLAGS} -fPIC -c service_internal.c service_yaml.o: service_yaml.c service.h @@ -126,5 +126,5 @@ zcfg2yaml: zcfg2yaml.c service.h lex.sr.o service_conf.tab.o service_conf.y main_conf_read.tab.o lex.cr.o response_print.o server_internal.o service_internal.o ${MS_FILE} ${YAML_FILE} g++ -g -O2 ${JSCFLAGS} ${RUBYCFLAGS} ${XML2CFLAGS} ${CFLAGS} -c zcfg2yaml.c -fno-common -DPIC -o zcfg2yaml.o - g++ ${XML2CFLAGS} ${CFLAGS} zcfg2yaml.o server_internal.o service_internal.o ${MS_FILE} response_print.o lex.cr.o lex.sr.o service_conf.tab.o main_conf_read.tab.o ${YAML_FILE} -o zcfg2yaml -L. ${LDFLAGS} + g++ ${XML2CFLAGS} ${CFLAGS} zcfg2yaml.o ulinet.o service_callback.o server_internal.o service_internal.o ${MS_FILE} response_print.o lex.cr.o lex.sr.o service_conf.tab.o main_conf_read.tab.o ${YAML_FILE} -o zcfg2yaml -L. ${LDFLAGS} install: zoo_loader.cgi Index: branches/prototype-v0/zoo-project/zoo-kernel/ZOOMakefile.opts.in =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/ZOOMakefile.opts.in (revision 851) +++ branches/prototype-v0/zoo-project/zoo-kernel/ZOOMakefile.opts.in (revision 854) @@ -115,5 +115,5 @@ CFLAGS=@SSH2_CPPFLAGS@ @GETTEXT_CFLAGS@ @PROJ_CPPFLAGS@ @RELY_ON_DB@ @DEB_DEF@ -fpic @OPENSSL_CFLAGS@ @UUID_CFLAGS@ ${FCGI_CFLAGS} ${YAML_CFLAGS} ${MACOS_CFLAGS} ${MS_CFLAGS} -I../../thirds/cgic206 -I. -DLINUX_FREE_ISSUE #-DDEBUG #-DDEBUG_SERVICE_CONF -LDFLAGS=-lzoo_service @SSH2_LDFLAGS@ @PROJ_LDFLAGS@ @DEFAULT_LIBS@ -L../../thirds/cgic206 -lcgic ${GDAL_LIBS} ${XML2LDFLAGS} ${PYTHONLDFLAGS} ${PERLLDFLAGS} ${PHPLDFLAGS} ${JAVALDFLAGS} ${JSLDFLAGS} ${FCGI_LDFLAGS} @OPENSSL_LDFLAGS@ @UUID_LDFLAGS@ ${MS_LDFLAGS} ${MACOS_LD_FLAGS} ${MACOS_LD_NET_FLAGS} ${YAML_LDFLAGS} ${OTBLDFLAGS} ${SAGA_LDFLAGS} ${MONO_LDFLAGS} @GETTEXT_LDFLAGS@ @JSON_LDFLAGS@ +LDFLAGS=-lzoo_service @SSH2_LDFLAGS@ @PROJ_LDFLAGS@ @DEFAULT_LIBS@ -L../../thirds/cgic206 -lcgic ${GDAL_LIBS} ${XML2LDFLAGS} ${PYTHONLDFLAGS} ${PERLLDFLAGS} ${PHPLDFLAGS} ${JAVALDFLAGS} ${JSLDFLAGS} ${FCGI_LDFLAGS} @OPENSSL_LDFLAGS@ @UUID_LDFLAGS@ ${MS_LDFLAGS} ${MACOS_LD_FLAGS} ${MACOS_LD_NET_FLAGS} ${YAML_LDFLAGS} ${OTBLDFLAGS} ${SAGA_LDFLAGS} ${MONO_LDFLAGS} @GETTEXT_LDFLAGS@ @JSON_LDFLAGS@ @XSLT_LDFLAGS@ DATAROOTDIR=@datarootdir@/zoo-project Index: branches/prototype-v0/zoo-project/zoo-kernel/caching.c =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/caching.c (revision 851) +++ branches/prototype-v0/zoo-project/zoo-kernel/caching.c (revision 854) @@ -64,4 +64,70 @@ /** + * Create a URL by appending every request header listed in the security + * section.This imply that the URL will contain any authentication + * informations that should be fowarded to the server from which de input + * was download. + * @param conf the main configuration maps + * @param request the URL to transform. + * @return a char* that contain the original URL plus potential header (only for + * hosts that are not shared). + * @warning Be sure to free the memory returned by this function. + */ +char* getFilenameForRequest(maps* conf, const char* request){ + map* passThrough=getMapFromMaps(conf,"security","attributes"); + map* targetHosts=getMapFromMaps(conf,"security","hosts"); + char* passedHeader[10]; + int cnt=0; + char *res=zStrdup(request); + char *toAppend=NULL; + if(passThrough!=NULL && targetHosts!=NULL){ + char *tmp=zStrdup(passThrough->value); + char *token, *saveptr; + token = strtok_r (tmp, ",", &saveptr); + int i; + if((strstr(targetHosts->value,"*")!=NULL || isProtectedHost(targetHosts->value,request)==1) && strncasecmp(getProvenance(conf,request),"SHARED",6)!=0){ + while (token != NULL){ + int length=strlen(token)+6; + char* tmp1=(char*)malloc(length*sizeof(char)); + map* tmpMap; + snprintf(tmp1,6,"HTTP_"); + int j; + for(j=0;token[j]!='\0';j++){ + if(token[j]!='-') + tmp1[5+j]=toupper(token[j]); + else + tmp1[5+j]='_'; + tmp1[5+j+1]='\0'; + } + tmpMap = getMapFromMaps(conf,"renv",tmp1); + if(tmpMap!=NULL){ + if(toAppend==NULL){ + toAppend=(char*)malloc((strlen(tmpMap->value)+1)*sizeof(char)); + sprintf(toAppend,"%s",tmpMap->value); + }else{ + char *tmp3=zStrdup(toAppend); + toAppend=(char*)realloc(toAppend,(strlen(tmpMap->value)+strlen(tmp3)+2)*sizeof(char)); + sprintf(toAppend,"%s,%s",tmp3,tmpMap->value); + free(tmp3); + } + } + free(tmp1); + cnt+=1; + token = strtok_r (NULL, ",", &saveptr); + } + } + free(tmp); + } + if(toAppend!=NULL){ + char *tmp3=zStrdup(res); + res=(char*)realloc(res,(strlen(tmp3)+strlen(toAppend)+1)*sizeof(char)); + sprintf(res,"%s%s",tmp3,toAppend); + free(tmp3); + free(toAppend); + } + return res; +} + +/** * Cache a file for a given request. * For each cached file, the are two files stored, a .zca and a .zcm containing @@ -80,39 +146,55 @@ map* tmp=getMapFromMaps(conf,"main","cacheDir"); if(tmp!=NULL){ - char* md5str=getMd5(request); + char* myRequest=getFilenameForRequest(conf,request); + char* md5str=getMd5(myRequest); + free(myRequest); char* fname=(char*)malloc(sizeof(char)*(strlen(tmp->value)+strlen(md5str)+6)); sprintf(fname,"%s/%s.zca",tmp->value,md5str); + zooLock* lck=lockFile(conf,fname,'w'); + if(lck!=NULL){ #ifdef DEBUG - fprintf(stderr,"Cache list : %s\n",fname); - fflush(stderr); + fprintf(stderr,"Cache list : %s\n",fname); + fflush(stderr); #endif - FILE* fo=fopen(fname,"w+"); - if(fo==NULL){ + FILE* fo=fopen(fname,"w+"); + if(fo==NULL){ #ifdef DEBUG - fprintf (stderr, "Failed to open %s for writing: %s\n",fname, strerror(errno)); + fprintf (stderr, "Failed to open %s for writing: %s\n",fname, strerror(errno)); #endif - filepath = NULL; - return; - } - fwrite(content,sizeof(char),length,fo); - fclose(fo); + filepath = NULL; + unlockFile(conf,lck); + return; + } + fwrite(content,sizeof(char),length,fo); + unlockFile(conf,lck); + fclose(fo); - if (filepath != NULL) { - strncpy(filepath, fname, max_path); - } - - sprintf(fname,"%s/%s.zcm",tmp->value,md5str); - fo=fopen(fname,"w+"); + if (filepath != NULL) { + strncpy(filepath, fname, max_path); + } + + sprintf(fname,"%s/%s.zcm",tmp->value,md5str); + fo=fopen(fname,"w+"); #ifdef DEBUG - fprintf(stderr,"MIMETYPE: %s\n",mimeType); + fprintf(stderr,"MIMETYPE: %s\n",mimeType); #endif - fwrite(mimeType,sizeof(char),strlen(mimeType),fo); - fclose(fo); - - free(md5str); - free(fname); + fwrite(mimeType,sizeof(char),strlen(mimeType),fo); + fclose(fo); + + sprintf(fname,"%s/%s.zcp",tmp->value,md5str); + fo=fopen(fname,"w+"); + char* origin=getProvenance(conf,request); +#ifdef DEBUG + fprintf(stderr,"ORIGIN: %s\n",mimeType); +#endif + fwrite(origin,sizeof(char),strlen(origin),fo); + fclose(fo); + + free(md5str); + free(fname); + } } else { - filepath = NULL; + filepath = NULL; } } @@ -129,5 +211,7 @@ map* tmpM=getMapFromMaps(conf,"main","cacheDir"); if(tmpM!=NULL){ - char* md5str=getMd5(request); + char* myRequest=getFilenameForRequest(conf,request); + char* md5str=getMd5(myRequest); + free(myRequest); #ifdef DEBUG fprintf(stderr,"MD5STR : (%s)\n\n",md5str); @@ -206,5 +290,5 @@ sprintf(xname,"Reference"); sprintf(bname,"body"); - sprintf(hname,"headers",i); + sprintf(hname,"headers"); sprintf(oname,"Order"); } @@ -280,6 +364,8 @@ free(tmpStr); }else{ - md5str=getMd5(tmp1->value); + char *myRequest=getFilenameForRequest(*m,tmp1->value); + md5str=getMd5(myRequest); request=zStrdup(tmp1->value); + free(myRequest); } char* fname=(char*)malloc(sizeof(char)*(strlen(tmp->value)+strlen(md5str)+6)); @@ -387,8 +473,10 @@ if(cached!=NULL){ - struct stat f_status; int s=stat(cached, &f_status); if(s==0){ + zooLock* lck=lockFile(*m,cached,'r'); + if(lck==NULL) + return -1; fcontent=(char*)malloc(sizeof(char)*(f_status.st_size+1)); FILE* f=fopen(cached,"rb"); @@ -398,8 +486,12 @@ fclose(f); addToMap(*content,"cache_file",cached); + unlockFile(*m,lck); } cached[strlen(cached)-1]='m'; s=stat(cached, &f_status); if(s==0){ + zooLock* lck=lockFile(*m,cached,'r'); + if(lck==NULL) + return -1; mimeType=(char*)malloc(sizeof(char)*(f_status.st_size+1)); FILE* f=fopen(cached,"rb"); @@ -407,6 +499,6 @@ mimeType[f_status.st_size]=0; fclose(f); - } - + unlockFile(*m,lck); + } }else{ addRequestToQueue(m,hInternet,url,true); @@ -439,5 +531,7 @@ if(tmp!=NULL){ map *c=getMap((*content),"xlink:href"); - char* md5str=getMd5(c->value); + char *myRequest=getFilenameForRequest(*m,c->value); + char* md5str=getMd5(myRequest); + free(myRequest); char* fname=(char*)malloc(sizeof(char)*(strlen(tmp->value)+strlen(md5str)+6)); sprintf(fname,"%s/%s.zca",tmp->value,md5str); Index: branches/prototype-v0/zoo-project/zoo-kernel/meta_sql.c =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/meta_sql.c (revision 851) +++ branches/prototype-v0/zoo-project/zoo-kernel/meta_sql.c (revision 854) @@ -297,6 +297,4 @@ */ elements* extractInput(maps* conf,OGRFeature *input){ - fprintf(stderr,"************************* %s %s %d\n\n",input->GetFieldAsString( 0 ),__FILE__,__LINE__); - fprintf(stderr,"************************* %s %s %d\n\n",input->GetFieldAsString( 1 ),__FILE__,__LINE__); elements* res=createElements(input->GetFieldAsString( 1 )); res->content=createMap("title",input->GetFieldAsString( 2 )); @@ -348,8 +346,4 @@ fillMetadata(conf,&res->metadata,output->GetFieldAsString( 0 )); fillAdditionalParameters(conf,&res->additional_parameters,output->GetFieldAsString( 0 )); - res->defaults=NULL; - res->supported=NULL; - res->child=NULL; - res->next=NULL; int ioCnt=fillLiteralData(conf,res,output,"Output"); if(ioCnt==0) @@ -361,5 +355,5 @@ free(nestedOutputsQuery); while( (noutput = noutputs->GetNextFeature()) != NULL ){ - elements* nout=extractInput(conf,noutput); + elements* nout=extractOutput(conf,noutput); addToElements(&res->child,nout); freeElements(&nout); @@ -413,8 +407,5 @@ elements* in=extractInput(conf,input); if(in!=NULL){ - if(s->inputs==NULL) - s->inputs=dupElements(in); - else - addToElements(&s->inputs,in); + addToElements(&s->inputs,in); freeElements(&in); free(in); @@ -431,10 +422,9 @@ while( (output = outputs->GetNextFeature()) != NULL ){ elements* in=extractOutput(conf,output); - if(s->outputs==NULL) - s->outputs=dupElements(in); - else + if(in!=NULL){ addToElements(&s->outputs,in); - freeElements(&in); - free(in); + freeElements(&in); + free(in); + } OGRFeature::DestroyFeature( output ); } @@ -463,6 +453,6 @@ service *), int minimal ){ int result=0; - _init_sql(conf,"metadb"); - if(getMapFromMaps(conf,"lenv","dbIssue")!=NULL) + result=_init_sql(conf,"metadb"); + if(getMapFromMaps(conf,"lenv","dbIssue")!=NULL || result < 0) return -1; // Fetch every services Index: branches/prototype-v0/zoo-project/zoo-kernel/mimetypes.h =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/mimetypes.h (revision 851) +++ branches/prototype-v0/zoo-project/zoo-kernel/mimetypes.h (revision 854) @@ -818,21 +818,22 @@ */ static map* getFileExtensionMap(const char* mimeType, bool* hasExt) { - - map* ext = createMap("extension", "txt"); - *hasExt = false; + map* ext = createMap("extension", "txt"); + *hasExt = false; - if (mimeType != NULL) { - for (int i = 0; i < NUM_MIME_TYPES; i++) { - if (strncmp(mimeType, MIME[i][M_Type], strlen(MIME[i][M_Type])) == 0) { - ext->value = zStrdup(MIME[i][M_Extension]); - *hasExt = true; - break; - } - } - if (*hasExt == false && strncmp(mimeType, "image/", 6) == 0) { - ext->value = zStrdup(strstr(mimeType, "/") + 1); - } - } - return ext; + if (mimeType != NULL) { + for (int i = 0; i < NUM_MIME_TYPES; i++) { + if(strncmp(mimeType, MIME[i][M_Type], strlen(MIME[i][M_Type])) == 0) { + free(ext->value); + ext->value = zStrdup(MIME[i][M_Extension]); + *hasExt = true; + break; + } + } + if (*hasExt == false && strncmp(mimeType, "image/", 6) == 0) { + free(ext->value); + ext->value = zStrdup(strstr(mimeType, "/") + 1); + } + } + return ext; } Index: branches/prototype-v0/zoo-project/zoo-kernel/request_parser.c =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/request_parser.c (revision 851) +++ branches/prototype-v0/zoo-project/zoo-kernel/request_parser.c (revision 854) @@ -1849,2 +1849,40 @@ } } + +/** + * Parse cookie contained in request headers. + * + * @param conf the conf maps containinfg the main.cfg + * @param cookie the + */ +void parseCookie(maps** conf,const char* cookie){ + char* tcook=zStrdup(cookie); + char *token, *saveptr; + token = strtok_r (tcook, "; ", &saveptr); + maps* res=createMaps("cookies"); + while (token != NULL){ + char *token1, *saveptr1, *name; + int i=0; + token1 = strtok_r (token, "=", &saveptr1); + while (token1 != NULL){ + if(i==0){ + name=zStrdup(token1); + i++; + } + else{ + if(res->content==NULL) + res->content=createMap(name,token1); + else + addToMap(res->content,name,token1); + free(name); + i=0; + } + token1 = strtok_r (NULL, "=", &saveptr1); + } + token = strtok_r (NULL, "; ", &saveptr); + } + addMapsToMaps(conf,res); + freeMaps(&res); + free(res); + free(tcook); +} Index: branches/prototype-v0/zoo-project/zoo-kernel/request_parser.h =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/request_parser.h (revision 851) +++ branches/prototype-v0/zoo-project/zoo-kernel/request_parser.h (revision 854) @@ -49,5 +49,6 @@ void checkValidValue(map*,map**,const char*,const char**,int); int validateRequest(maps**,service*,map*,maps**,maps**,HINTERNET*); - + void parseCookie(maps**,const char*); + #ifdef __cplusplus } Index: branches/prototype-v0/zoo-project/zoo-kernel/response_print.c =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/response_print.c (revision 851) +++ branches/prototype-v0/zoo-project/zoo-kernel/response_print.c (revision 854) @@ -743,5 +743,13 @@ } -int addMetadata(map* meta,xmlDocPtr doc,xmlNodePtr nc,xmlNsPtr ns_ows,xmlNsPtr ns_xlink){ +/** + * Add a Metadata node to any existing node. + * @param meta the map defining the additional parameters + * @param doc the XML document used + * @param nb the node to add the additional parameters + * @param ns_ows the OWS namespace + * @param ns_xlink the xlink namespace + */ +void addMetadata(map* meta,xmlDocPtr doc,xmlNodePtr nc,xmlNsPtr ns_ows,xmlNsPtr ns_xlink){ int hasTitle=-1; int hasValue=-1; @@ -817,14 +825,30 @@ } - -int addAdditionalParameters(map* meta,xmlDocPtr doc,xmlNodePtr nc,xmlNsPtr ns_ows,xmlNsPtr ns_xlink,int fromDb){ +/** + * Add AdditionalParameters nodes to any existing node. + * @param meta the map defining the additional parameters + * @param doc the XML document used + * @param nb the node to add the additional parameters + * @param ns_ows the OWS namespace + * @param ns_xlink the xlink namespace + * @param fromDb 1 if the metadata has been extracted from the metadb, + * 0 otherwise + */ +void addAdditionalParameters(map* meta,xmlDocPtr doc,xmlNodePtr nc,xmlNsPtr ns_ows,xmlNsPtr ns_xlink,int fromDb){ int hasTitle=-1; int hasValue=-1; + int toAddAtEnd=-1; + int cnt=0; + xmlNodePtr* ncr=NULL; xmlNodePtr nc1; map* oMeta=meta; int isAdditionalParameters=-1; - //if(count(oMeta)>=2){ int level=0; map* test=getMap(meta,"title"); + map* otitle=getMap(meta,"title"); + map* length=getMap(meta,"length"); + int len=0; + char *ctitle=NULL; + if(test!=NULL) level+=1; @@ -837,10 +861,9 @@ if(count(oMeta)>level+1) isAdditionalParameters=1; - //} - char *ctitle=NULL; + while(meta!=NULL){ - if(hasTitle<0) - if(hasValue<0) - nc1 = xmlNewNode(ns_ows, BAD_CAST "AdditionalParameters"); + if(hasTitle<0 && hasValue<0){ + nc1 = xmlNewNode(ns_ows, BAD_CAST "AdditionalParameters"); + } if(strncasecmp(meta->name,"title",5)==0 || strcasecmp(meta->name,"href")==0 || @@ -851,8 +874,12 @@ hasTitle=1; if(ctitle!=NULL && strcasecmp(meta->value,ctitle)!=0){ - xmlAddChild(nc,nc1); - nc1 = xmlNewNode(ns_ows, BAD_CAST "AdditionalParameters"); - free(ctitle); - ctitle=NULL; + xmlNodePtr ncTmp = xmlDocCopyNodeList(doc,nc1); + xmlAddChild(nc,ncTmp); + xmlFreeNode(nc1); + toAddAtEnd=1; + cnt++; + nc1 = xmlNewNode(ns_ows, BAD_CAST "AdditionalParameters"); + free(ctitle); + ctitle=NULL; } if(ctitle==NULL){ @@ -865,4 +892,6 @@ free(ctitle); ctitle=zStrdup(meta->value); + }else{ + xmlNewNsProp(nc1,ns_xlink,BAD_CAST meta->name,BAD_CAST meta->value); } }else{ @@ -892,11 +921,17 @@ meta=meta->next; if(hasTitle<0){ - xmlAddChild(nc,nc1); + //xmlAddChild(nc,nc1); hasValue=1; - }else - free(ctitle); - } - if(oMeta!=NULL && hasValue<0 && nc1!=NULL){ + }/*else + if(ctitle!=NULL) + free(ctitle);*/ + } + if(length!=NULL) + len=atoi(length->value); + if(otitle!=NULL) + len=1; + if(cntname,"uom")==0) hasUOM1=true; @@ -1428,6 +1464,7 @@ if(hasUOM1==false && vid==0){ xmlFreeNode(nc5); - if(datatype==1) + if(datatype==1){ xmlFreeNode(nc4); + } } else @@ -1440,6 +1477,7 @@ if(datatype!=1 && default1<0){ xmlFreeNode(nc5); - if(datatype!=2) + if(datatype!=2){ xmlFreeNode(nc4); + } } @@ -1603,9 +1641,11 @@ if(vid==0 || oI!=4) xmlAddChild(nc5,nc6); - else + else{ xmlFreeNode(nc6); + } } - else + else{ xmlFreeNode(nc6); + } } tmp1=tmp1->next; @@ -1634,5 +1674,4 @@ } else{ - xmlFreeNode(nc4); xmlAddChild(nc3,nc5); } @@ -1648,6 +1687,7 @@ if(hasSupported==0){ - if(datatype==0 && vid!=0) + if(datatype==0 && vid!=0){ xmlFreeNode(nc4); + } xmlFreeNode(nc5); } @@ -2678,5 +2718,11 @@ toto=getMap(tmpI->content,"asReference"); #ifdef USE_MS - if(toto!=NULL && strcasecmp(toto->value,"true")==0 && (testMap==NULL || strncasecmp(testMap->value,"true",4)!=0) ) + restartNoMS: + map* geodatatype=getMap(tmpI->content,"geodatatype"); + + if(toto!=NULL && strcasecmp(toto->value,"true")==0 && + (testMap==NULL || + strncasecmp(testMap->value,"true",4)!=0 || + (geodatatype!=NULL && strcasecmp(geodatatype->value,"other")==0) ) ) #else if(toto!=NULL && strcasecmp(toto->value,"true")==0) @@ -2699,7 +2745,7 @@ } + char *file_path=NULL; if(gfile==NULL) { map *ext=getMap(tmpI->content,"extension"); - char *file_path; char file_ext[32]; @@ -2716,5 +2762,5 @@ free(file_name); file_name=(char*)malloc((strlen(s->name)+strlen(usid->value)+strlen(file_ext)+strlen(tmpI->name)+45)*sizeof(char)); - sprintf(file_name,"%s_%s_%s_%d.%s",s->name,tmpI->name,usid->value,itn,file_ext); + sprintf(file_name,"ZOO_DATA_%s_%s_%s_%d.%s",s->name,tmpI->name,usid->value,itn,file_ext); itn++; file_path=(char*)malloc((strlen(tmp1->value)+strlen(file_name)+2)*sizeof(char)); @@ -2730,5 +2776,4 @@ return; } - free(file_path); toto=getMap(tmpI->content,"value"); @@ -2744,10 +2789,9 @@ } fclose(ofile); - } map *tmp2=getMapFromMaps(m,"main","tmpUrl"); map *tmp3=getMapFromMaps(m,"main","serverAddress"); - char *file_url; + char *file_url=NULL; if(strncasecmp(tmp2->value,"http://",7)==0 || strncasecmp(tmp2->value,"https://",8)==0){ @@ -2759,6 +2803,18 @@ } addToMap(tmpI->content,"Reference",file_url); - free(file_name); - free(file_url); + /*maps* curs=tmpI; + curs=curs->next; + map* test=getMap(tmpI->content,"replicateStorageNext"); + if(test!=NULL && strncasecmp(test->value,"true",4)==0) + while(curs!=NULL){ + addToMap(curs->content,"storage",file_path); + curs=curs->next; + } + if(file_path!=NULL) + free(file_path);*/ + if(file_name!=NULL) + free(file_name); + if(file_url!=NULL) + free(file_url); file_name=NULL; } @@ -2766,5 +2822,9 @@ else{ if(testMap!=NULL){ + setMapInMaps(m,"lenv","state","out"); setReferenceUrl(m,tmpI); + geodatatype=getMap(tmpI->content,"geodatatype"); + if(geodatatype!=NULL && strcasecmp(geodatatype->value,"other")==0) + goto restartNoMS; } } Index: branches/prototype-v0/zoo-project/zoo-kernel/response_print.h =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/response_print.h (revision 851) +++ branches/prototype-v0/zoo-project/zoo-kernel/response_print.h (revision 854) @@ -220,6 +220,6 @@ void printOutputDefinitions(xmlDocPtr,xmlNodePtr,xmlNsPtr,xmlNsPtr,elements*,maps*,const char*); void printStatusInfo(maps*,map*,char*); - int addAdditionalParameters(map*,xmlDocPtr,xmlNodePtr,xmlNsPtr,xmlNsPtr,int); - int addMetadata(map*,xmlDocPtr,xmlNodePtr,xmlNsPtr,xmlNsPtr); + void addAdditionalParameters(map*,xmlDocPtr,xmlNodePtr,xmlNsPtr,xmlNsPtr,int); + void addMetadata(map*,xmlDocPtr,xmlNodePtr,xmlNsPtr,xmlNsPtr); void outputResponse(service*,maps*,maps*,map*,int,maps*,int); Index: branches/prototype-v0/zoo-project/zoo-kernel/server_internal.c =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/server_internal.c (revision 851) +++ branches/prototype-v0/zoo-project/zoo-kernel/server_internal.c (revision 854) @@ -26,4 +26,5 @@ #include "service_internal.h" #include "response_print.h" +#include "service_callback.h" #include "mimetypes.h" #ifndef WIN32 @@ -444,5 +445,6 @@ if(cSize!=NULL){ length=atoi(cSize->value); - } + }else + length=strlen(cValue->value); writeFile(val,cValue->value,length); setMapArray(cMap,"cache_file",k,val); @@ -466,4 +468,5 @@ free(val); } + addToMap(inputs->content,"byValue","true"); } inputs=inputs->next; @@ -980,5 +983,5 @@ if(isRunning(conf,pid)>0){ if(strncasecmp(req,"GetResult",strlen(req))==0){ - errorException (conf, _("The result for the requested JobID has not yet been generated. (1)"), + errorException (conf, _("The result for the requested JobID has not yet been generated. The service is currently running."), "ResultNotReady", pid); return; @@ -1008,9 +1011,11 @@ printf("%s",result); fflush(stdout); + free(sid); freeMap(&statusInfo); free(statusInfo); + free(result); return; }else{ - errorException (conf, _("The result for the requested JobID has not yet been generated. (2)"), + errorException (conf, _("The result for the requested JobID has not yet been generated. The service ends but it still needs to produce the outputs."), "ResultNotReady", pid); freeMap(&statusInfo); @@ -1104,4 +1109,9 @@ removeService(conf,pid); #endif + /* + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + invokeCallback(conf,NULL,NULL,7,1); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + */ map* statusInfo=createMap("JobID",pid); addToMap(statusInfo,"Status","Dismissed"); Index: branches/prototype-v0/zoo-project/zoo-kernel/service.c =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/service.c (revision 851) +++ branches/prototype-v0/zoo-project/zoo-kernel/service.c (revision 854) @@ -1172,17 +1172,17 @@ tmp->name=zStrdup(cursor->name); tmp->content=NULL; - addMapToMap(&tmp->content,e->content); + addMapToMap(&tmp->content,cursor->content); tmp->metadata=NULL; - addMapToMap(&tmp->metadata,e->metadata); + addMapToMap(&tmp->metadata,cursor->metadata); tmp->additional_parameters=NULL; - addMapToMap(&tmp->additional_parameters,e->additional_parameters); - if(e->format!=NULL) - tmp->format=zStrdup(e->format); + addMapToMap(&tmp->additional_parameters,cursor->additional_parameters); + if(cursor->format!=NULL) + tmp->format=zStrdup(cursor->format); else tmp->format=NULL; - if(e->defaults!=NULL){ + if(cursor->defaults!=NULL){ tmp->defaults=(iotype*)malloc(IOTYPE_SIZE); tmp->defaults->content=NULL; - addMapToMap(&tmp->defaults->content,e->defaults->content); + addMapToMap(&tmp->defaults->content,cursor->defaults->content); tmp->defaults->next=NULL; #ifdef DEBUG @@ -1192,10 +1192,10 @@ }else tmp->defaults=NULL; - if(e->supported!=NULL){ + if(cursor->supported!=NULL && cursor->supported->content!=NULL){ tmp->supported=(iotype*)malloc(IOTYPE_SIZE); tmp->supported->content=NULL; - addMapToMap(&tmp->supported->content,e->supported->content); + addMapToMap(&tmp->supported->content,cursor->supported->content); tmp->supported->next=NULL; - iotype *tmp2=e->supported->next; + iotype *tmp2=cursor->supported->next; while(tmp2!=NULL){ addMapToIoType(&tmp->supported,tmp2->content); Index: branches/prototype-v0/zoo-project/zoo-kernel/service_callback.c =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/service_callback.c (revision 851) +++ branches/prototype-v0/zoo-project/zoo-kernel/service_callback.c (revision 854) @@ -30,5 +30,15 @@ #include "service_json.h" #include "service_internal_ms.h" +#include "sqlapi.h" #include +#include +#include +#include +#include + +#include +#include +#include +#include #ifdef __cplusplus @@ -36,7 +46,34 @@ #endif + /** + * Parameter definition to be used for sending parameters to a thread. + */ + typedef struct { + maps *conf; //!< the main configuration file + map *url; //!< the callback url maps + json_object *res;//!< the JSON object to post + int step; //!< the current step [0,6] + int state; //!< the current state [0,1] + } local_params; + + /** + * Number of threads + */ int nbThreads=0; + /** + * Current step + */ int cStep=0; + /** + * Is there any ongoing HTTP request + */ + int isOngoing=0; + /** + * Threads array + */ pthread_t* myThreads=NULL; + /** + * Steps array + */ bool steps[7][2]={ {false,false}, @@ -48,4 +85,8 @@ {false,false} }; + /** + * Arguments array to give to the _invokeCallback thread's function + */ + local_params** local_arguments; /** @@ -71,45 +112,4 @@ } - /** - * Parameter definition to be used for sending parameters to a thread. - */ - typedef struct { - maps *conf; //!< the main configuration file - map *url; //!< the callback url maps - json_object *res;//!< the JSON object to post - int step; //!< the current step [0,6] - int state; //!< the current state [0,1] - } local_params; - - /** - * Verify if the URL should use a shared cache or not. - * - * In case the security section contains a key named "shared", then if the - * domain listed in the shared key are contained in the url given as parameter - * then it return "SHARED" in other cases, it returns "OTHER". - * - * @param conf the main configuration file maps - * @param url the URL to evaluate - * @return a string "SHARED" in case the host is in a domain listed in the - * shared key, "OTHER" in other cases. - */ - char* getProvenance(maps* conf,const char* url){ - map* sharedCache=getMapFromMaps(conf,"security","shared"); - if(sharedCache!=NULL){ - char *res=NULL; - char *hosts=sharedCache->value; - char *curs=strtok(hosts,","); - while(curs!=NULL){ - if(strstr(url,curs)==NULL) - res="OTHER"; - else{ - res="SHARED"; - return res; - } - } - return res; - } - return "OTHER"; - } /** @@ -133,21 +133,44 @@ hInternet.waitingRequests[0] = zStrdup(URL); free(URL); - fprintf(stderr,"************************* From thread %d %s %d: REQUEST PARAMETERS\n",pthread_self(),__FILE__,__LINE__); +#ifdef CALLBACK_DEBUG + fprintf(stderr,"************************* From thread %d %s %d: REQUEST PARAMETERS cStep %d %d\n",pthread_self(),__FILE__,__LINE__,cStep,isOngoing); fprintf(stderr," * JSON: [%s] \n",jsonStr); fprintf(stderr," * URL: %s/ \n\n",hInternet.waitingRequests[0]); fprintf(stderr,"************************* From thread %d %s %d: REQUEST PARAMETERS\n",pthread_self(),__FILE__,__LINE__); - while( cStep!=7 && +#endif + while( (arg->step!=7 || isOngoing>0) && ( cStep!=arg->step || (arg->state!=0 && steps[arg->step][0]==false) ) ){ - sleep(1); - } + struct timespec tv; + tv.tv_sec = 0; + tv.tv_nsec = (long) 5*1e+9; + nanosleep(&tv, &tv); + //sleep(1); + } + isOngoing=1; +#ifdef CALLBACK_DEBUG fprintf(stderr,"************************* From thread %d %s %d: REQUEST START\n\n",pthread_self(),__FILE__,__LINE__); int i=0; for(i=0;i<7;i++){ fprintf(stderr,"%d) %d %d\n",i,steps[i][0],steps[i][1]); - fflush(stderr); - } - fprintf(stderr,"************************* From thread %d %s %d: REQUEST START\n",pthread_self(),__FILE__,__LINE__); + } +#endif + const struct tm *tm; + size_t len; + time_t now; + char *tmp1; + map *tmpStatus; + + now = time ( NULL ); + tm = localtime ( &now ); + + tmp1 = (char*)malloc((TIME_SIZE+1)*sizeof(char)); + len = strftime ( tmp1, TIME_SIZE, "%Y-%m-%dT%I:%M:%SZ", tm ); + +#ifdef CALLBACK_DEBUG + fprintf(stderr,"************************* From thread %d %s %d: REQUEST START (%s)\n",pthread_self(),__FILE__,__LINE__,tmp1); fflush(stderr); +#endif + free(tmp1); res1 = InternetOpenUrl (&hInternet, hInternet.waitingRequests[0], @@ -158,6 +181,13 @@ //curl_easy_setopt(hInternet.ihandle[hInternet.nb].handle, CURLOPT_VERBOSE, 1);x processDownloads(&hInternet); - fprintf(stderr,"************************* From thread %d %s %d: REQUEST END\n\n",pthread_self(),__FILE__,__LINE__); - fflush(stderr); + now = time ( NULL ); + tm = localtime ( &now ); + tmp1 = (char*)malloc((TIME_SIZE+1)*sizeof(char)); + len = strftime ( tmp1, TIME_SIZE, "%Y-%m-%dT%I:%M:%SZ", tm ); + +#ifdef CALLBACK_DEBUG + fprintf(stderr,"************************* From thread %d %s %d: REQUEST END (%s)\n\n",pthread_self(),__FILE__,__LINE__,tmp1); +#endif + free(tmp1); char *tmp = (char *) malloc ((hInternet.ihandle[0].nDataLen + 1) * sizeof (char)); @@ -177,7 +207,8 @@ json_object_put(arg->res); InternetCloseHandle(&hInternet); + isOngoing=0; if(cStep==0 || cStep==6 || arg->state==1) cStep=arg->step+1; - steps[arg->step][arg->state]=true; +#ifdef CALLBACK_DEBUG fprintf(stderr,"************************* From thread %d %s %d: RESPONSE CONTENT\n",pthread_self(),__FILE__,__LINE__); for(i=0;i<7;i++){ @@ -187,6 +218,10 @@ fprintf(stderr,"************************* From thread %d %s %d\n\n",pthread_self(),__FILE__,__LINE__); fflush(stderr); +#endif + steps[arg->step][arg->state]=true; free(tmp); - free(args); + //free(args); + fprintf(stderr,"************************* From thread %d %s %d: EXIT\n\n",pthread_self(),__FILE__,__LINE__); + fflush(stderr); pthread_exit(NULL); } @@ -237,14 +272,14 @@ tmp1 = (char*)malloc((TIME_SIZE+1)*sizeof(char)); - - len = strftime ( tmp1, TIME_SIZE, "%Y-%m-%dT%I:%M:%SZ", tm ); + len = strftime ( tmp1, TIME_SIZE, "%Y-%m-%dT%H:%M:%SZ", tm ); json_object *jsStr0=json_object_new_string(tmp1); json_object_object_add(res,"datetime",jsStr0); - + free(tmp1); + switch(step){ case 0: { // Create a new analyze maps* lenv=getMaps(conf,"lenv"); - sid=getMapFromMaps(conf,"lenv","xrequest"); + sid=getMapFromMaps(conf,"renv","xrequest"); if(sid!=NULL){ json_object *jsStr=json_object_new_string(sid->value); @@ -256,10 +291,24 @@ json_object_object_add(res,"process_identifier",jsStr); } + // Save the Execute request on disk + map* tmpPath=getMapFromMaps(conf,"main","tmpPath"); + map* req=getMapFromMaps(conf,"renv","xrequest"); + sid=getMapFromMaps(conf,"lenv","usid"); + char* executePath=(char*)malloc((strlen(tmpPath->value)+strlen(sid->value)+14)*sizeof(char)); + sprintf(executePath,"%s/execute_%s.xml",tmpPath->value,sid->value); + FILE* saveExecute=fopen(executePath,"wb"); + fwrite(req->value,1,strlen(req->value)*sizeof(char),saveExecute); + fflush(saveExecute); + fclose(saveExecute); + setMapInMaps(conf,"lenv","execute_file",executePath); + free(executePath); break; } + case 1: { // Fetching data inputs maps* curs=inputs; - char *keys[8][2]={ + dumpMaps(curs); + char *keys[10][2]={ { "href", @@ -287,10 +336,18 @@ }, { - "ref_wfs_link", - "ref_wfs_link" + "ref_wcs_link", + "ref_wcs_link" + }, + { + "ref_wcs_preview_link", + "ref_wcs_preview_link" }, { "geodatatype", "datatype" + }, + { + "wcs_extent", + "boundingbox" } }; @@ -300,6 +357,11 @@ sid=getMap(curs->content,"ref_wms_link"); json_object *res2=json_object_new_object(); - if(tmpMap!=NULL && sid==NULL){ - addToMap(curs->content,"generated_file",tmpMap->value); + if(tmpMap!=NULL){ + if(sid==NULL){ + addToMap(curs->content,"generated_file",tmpMap->value); + addToMap(curs->content,"storage",tmpMap->value); + } + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + dumpMap(curs->content); struct stat buf; char timeStr[ 100 ] = ""; @@ -314,10 +376,8 @@ } setReferenceUrl(conf,curs); - //outputMapfile(conf,curs); - dumpMaps(curs); } int i=0; int hasRef=-1; - for(;i<8;i++){ + for(;i<10;i++){ sid=getMap(curs->content,keys[i][0]); if(sid!=NULL){ @@ -340,5 +400,142 @@ break; } + case 2: { + // Update the execute request stored on disk at step 1,1 to modify the references used. + if(state==0){ + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + fflush(stderr); + maps* curs=inputs; + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + fflush(stderr); + xmlInitParser(); + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + fflush(stderr); + map* xmlPath=getMapFromMaps(conf,"lenv","execute_file"); + dumpMap(xmlPath); + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + fflush(stderr); + while(curs!=NULL){ + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + fflush(stderr); + dumpMap(curs->content); + //map* bvMap=getMap(curs->content,"byValue"); + // TODO handle mapArray + //if(bvMap!=NULL && strncasecmp(bvMap->value,"true",4)==0){ + if(getMap(curs->content,"href")==NULL && getMap(curs->content,"mimeType")!=NULL){ + map* tmpMap=getMap(curs->content,"cache_file"); + addToMap(curs->content,"generated_file",tmpMap->value); + addToMap(curs->content,"storage",tmpMap->value); + setReferenceUrl(conf,curs); + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + fflush(stderr); + dumpMap(curs->content); + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + fflush(stderr); + const char *params[5]; + int xmlLoadExtDtdDefaultValue; + int hasFile=-1; + map* xslPath=getMapFromMaps(conf,"callback","template"); + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + fflush(stderr); + dumpMap(xslPath); + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + fflush(stderr); + map* filePath=getMap(curs->content,"ref_wfs_link"); + if(filePath==NULL) + filePath=getMap(curs->content,"ref_wcs_link"); + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + fflush(stderr); + dumpMap(filePath); + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + fflush(stderr); + char* inputName=curs->name; + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + fflush(stderr); + if(xslPath==NULL || xmlPath==NULL || filePath==NULL) + break; + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + fflush(stderr); + char *tmpParam=(char*)malloc((strlen(curs->name)+11)*sizeof(char)); + char *tmpParam1=(char*)malloc((strlen(filePath->value)+11)*sizeof(char)); + sprintf(tmpParam,"string(\"%s\")",curs->name); + sprintf(tmpParam1,"string(\"%s\")",filePath->value); + params[0]="attr"; + params[1]=tmpParam; + params[2]="value"; + params[3]=tmpParam1;//filePath->value; + params[4]=NULL; + fprintf(stderr, "## XSLT PARAMETERS ATTR: %s VALUE: %s \n", + tmpParam,tmpParam1); + xmlSubstituteEntitiesDefault(1); + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + fflush(stderr); + xmlLoadExtDtdDefaultValue = 0; + xsltStylesheetPtr cur = NULL; + xmlDocPtr doc, res; + cur = xsltParseStylesheetFile(BAD_CAST xslPath->value); + doc = xmlParseFile(xmlPath->value); + fflush(stderr); + res = xsltApplyStylesheet(cur, doc, params); + xmlChar *xmlbuff; + int buffersize; + xmlDocDumpFormatMemory(res, &xmlbuff, &buffersize, 1); + // Store the executeRequest in file again + free(tmpParam); + free(tmpParam1); + fprintf(stderr," # Request / XSLT: %s\n",xmlbuff); + fflush(stderr); + FILE* saveExecute=fopen(xmlPath->value,"wb"); + fwrite(xmlbuff,1,buffersize,saveExecute); + fflush(saveExecute); + fclose(saveExecute); + xmlFree(xmlbuff); + xmlFreeDoc(doc); + xsltFreeStylesheet(cur); + } + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + fflush(stderr); + curs=curs->next; + } + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + fflush(stderr); + xmlCleanupParser(); + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + fflush(stderr); + FILE* f0=fopen(xmlPath->value,"rb"); + if(f0!=NULL){ + long flen; + char *fcontent; + fseek (f0, 0, SEEK_END); + flen = ftell (f0); + fseek (f0, 0, SEEK_SET); + fcontent = (char *) malloc ((flen + 1) * sizeof (char)); + fread(fcontent,flen,1,f0); + fcontent[flen]=0; + fclose(f0); + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + fflush(stderr); + map *schema=getMapFromMaps(conf,"database","schema"); + map* sid=getMapFromMaps(conf,"lenv","usid"); + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + fflush(stderr); + char *req=(char*)malloc((flen+strlen(schema->value)+strlen(sid->value)+66)*sizeof(char)); + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + fflush(stderr); + sprintf(req,"UPDATE %s.services set request_execute_content=$$%s$$ WHERE uuid=$$%s$$",schema->value,fcontent,sid->value); + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + fflush(stderr); + execSql(conf,1,req); + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + fflush(stderr); + free(fcontent); + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + fflush(stderr); + free(req); + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + fflush(stderr); + } + } + // Uploading data input to cluster maps* in=getMaps(conf,"uploadQueue"); @@ -371,4 +568,5 @@ break; } + case 3: { // Generating job script @@ -380,4 +578,5 @@ break; } + case 4: { // Submitting job to cluster @@ -389,10 +588,11 @@ break; } + case 5: { // Downloading process outputs from cluster //json_object* in=mapsToJson(outputs); - dumpMaps(outputs); maps* curs=outputs; - char *keys[8][2]={ + dumpMaps(curs); + char *keys[10][2]={ { "Reference", @@ -400,9 +600,9 @@ }, { - "storage", + "generated_file", "cachefile" }, { - "fmimeType", + "mimeType", "mimetype" }, @@ -412,4 +612,12 @@ }, { + "geodatatype", + "datatype" + }, + { + "wms_extent", + "boundingbox" + }, + { "ref_wms_link", "ref_wms_link" @@ -420,20 +628,40 @@ }, { + "ref_wcs_link", + "ref_wcs_preview_link" + }, + { "ref_wfs_link", "ref_wfs_link" - }, - { - "geodatatype", - "datatype" } + }; + char* specifics[5][2]={ + { + "download_link", + "ref_download_link" + }, + { + "wms_link", + "ref_wms_link" + }, + { + "wfs_link", + "ref_wfs_link" + }, + { + "wcs_link", + "ref_wcs_link" + }, + { + "wcs_link", + "ref_wcs_preview_link" + } }; json_object *res1=json_object_new_object(); while(curs!=NULL){ - map* tmpMap=getMap(curs->content,"cache_file"); - sid=getMap(curs->content,"ref_wms_link"); json_object *res2=json_object_new_object(); int i=0; int hasRef=-1; - for(;i<8;i++){ + for(;i<10;i++){ sid=getMap(curs->content,keys[i][0]); if(sid!=NULL){ @@ -448,23 +676,55 @@ else{ maps* curs0=curs->child; - while(curs0!=NULL){ - json_object *res3=json_object_new_object(); - int i0=0; - int hasRef0=-1; - for(;i0<8;i0++){ - sid=getMap(curs0->content,keys[i0][0]); - if(sid!=NULL){ - json_object *jsStr=json_object_new_string(sid->value); - json_object_object_add(res3,keys[i0][1],jsStr); - //if(i0==0) - hasRef0=1; + int i=0; + int bypass=-1; + for(i=0;i<5;i++){ + maps* specificMaps; + if((specificMaps=getMaps(curs0,specifics[i][0]))!=NULL){ + int hasRef0=-1; + int i0=0; + for(;i0<6;i0++){ + sid=getMap(specificMaps->content,keys[i0][0]); + if(sid!=NULL){ + json_object *jsStr=json_object_new_string(sid->value); + if(i0==0){ + json_object_object_add(res2,specifics[i][1],jsStr); + } + else + json_object_object_add(res2,keys[i0][1],jsStr); + hasRef0=1; + bypass=1; + if(i==1){ + struct stat buf; + char timeStr[ 100 ] = ""; + if (stat(sid->value, &buf)==0){ + strftime(timeStr, 100, "%d-%m-%Y %H:%M:%S", localtime( &buf.st_mtime)); + json_object *jsStr=json_object_new_string(timeStr); + json_object_object_add(res2,"creation_date",jsStr); + } + } + } + } + } + } + if(bypass<0) + while(curs0!=NULL){ + json_object *res3=json_object_new_object(); + int i0=0; + int hasRef0=-1; + for(;i0<10;i0++){ + sid=getMap(curs0->content,keys[i0][0]); + if(sid!=NULL){ + json_object *jsStr=json_object_new_string(sid->value); + json_object_object_add(res3,keys[i0][1],jsStr); + //if(i0==0) + hasRef0=1; + } } + if(hasRef0<0) + json_object_put(res3); + else + json_object_object_add(res2,curs0->name,res3); + curs0=curs0->next; } - if(hasRef0<0) - json_object_put(res3); - else - json_object_object_add(res2,curs0->name,res3); - curs0=curs0->next; - } json_object_object_add(res1,curs->name,res2); } @@ -474,13 +734,55 @@ break; } + case 6: { // Finalize HPC - sid=getMapFromMaps(conf,"lenv","local_script"); - if(sid!=NULL){ - json_object *jsStr=json_object_new_string(sid->value); - json_object_object_add(res,"inputs",jsStr); - } + char *keys[6][2]={ + { + "SubmitTime", + "hpc_submission_date" + }, + { + "JobId", + "hpc_job_identifier" + }, + { + "JobName", + "hpc_job_name" + }, + { + "StartTime", + "hpc_start_date" + }, + { + "EndTime", + "hpc_end_date" + }, + { + "JobState", + "hpc_status" + } + }; + int i=0; + if(getMaps(conf,"henv")!=NULL){ + for(i=0;i<6;i++){ + sid=getMapFromMaps(conf,"henv",keys[i][0]); + if(sid!=NULL){ + json_object *jsStr=json_object_new_string(sid->value); + json_object_object_add(res,keys[i][1],jsStr); + } + } + } + if((sid=getMapFromMaps(conf,"henv","billing_nb_cpu"))!=NULL){ + json_object *jsStr=json_object_new_string(sid->value); + json_object_object_add(res,"hpc_cpu_usage",jsStr); + }else{ + json_object *jsStr=json_object_new_string("1"); + json_object_object_add(res,"hpc_cpu_usage",jsStr); + } + json_object *jsStr=json_object_new_string("succeeded"); + json_object_object_add(res,"wps_status",jsStr); break; } + case 7: { // Error or Dismiss @@ -490,5 +792,9 @@ json_object_object_add(res,"message",jsStr); } - json_object *jsStr=json_object_new_string("failed"); + json_object *jsStr; + if(state==1) + jsStr=json_object_new_string("dismissed"); + else + jsStr=json_object_new_string("failed"); json_object_object_add(res,"wps_status",jsStr); break; @@ -498,11 +804,15 @@ } } - - local_params* argumentsA=(local_params*)malloc(MAPS_SIZE+MAP_SIZE+sizeof(json_object*)+(2*sizeof(int))); - argumentsA->conf=conf; - argumentsA->url=url; - argumentsA->res=res; - argumentsA->step=step; - argumentsA->state=state; + + if(local_arguments==NULL) + local_arguments=(local_params**)malloc(sizeof(local_params*)); + else + local_arguments=(local_params**)realloc(local_arguments,(nbThreads+1)*sizeof(local_params*)); + local_arguments[nbThreads]=(local_params*)malloc(MAPS_SIZE+MAP_SIZE+sizeof(json_object*)+(2*sizeof(int))); + local_arguments[nbThreads]->conf=conf; + local_arguments[nbThreads]->url=url; + local_arguments[nbThreads]->res=res; + local_arguments[nbThreads]->step=step; + local_arguments[nbThreads]->state=state; //pthread_t p1; if(myThreads==NULL) @@ -510,5 +820,5 @@ else myThreads=(pthread_t*)realloc(myThreads,(nbThreads+1)*sizeof(pthread_t)); - if(pthread_create(&myThreads[nbThreads], NULL, _invokeCallback, (void*)argumentsA)==-1){ + if(pthread_create(&myThreads[nbThreads], NULL, _invokeCallback, (void*)local_arguments[nbThreads])==-1){ setMapInMaps(conf,"lenv","message",_("Unable to create a new thread")); return false; @@ -519,9 +829,16 @@ } + /** + * Wait for the threads to end then, clean used memory. + */ void cleanupCallbackThreads(){ int i=0; for(i=0;ifilename=(char*)malloc((strlen(filename)+len)*sizeof(char)); + sprintf(myLock->filename,"%s.lock",filename); + s=stat(myLock->filename, &f_status); + if(s==0 && mode!='r'){ + if(itnfilename); + goto retryLockFile; + }else{ + free(myLock->filename); + free(myLock); + return NULL; + } + }else{ + char local_mode[3]; + memset(local_mode,0,3); + if(mode=='w') + sprintf(local_mode,"%c+",mode); + else + sprintf(local_mode,"%c",mode); + myLock->lockfile=fopen(myLock->filename,local_mode); + char tmp[512]; + sprintf(tmp,"%d",getpid()); + if(myLock->lockfile==NULL){ + myLock->lockfile=fopen(myLock->filename,"w+"); + fwrite(tmp,sizeof(char),strlen(tmp),myLock->lockfile); + fflush(myLock->lockfile); + fclose(myLock->lockfile); + myLock->lockfile=fopen(myLock->filename,local_mode); + }/*else + fprintf(stderr,"%s %d %d\n",__FILE__,__LINE__,(myLock->lockfile==NULL));*/ + if(mode!='r'){ + fwrite(tmp,sizeof(char),strlen(tmp),myLock->lockfile); + fflush(myLock->lockfile); + } + int cnt=0; + if(mode=='r'){ + myLock->lock.l_type = F_RDLCK; + }else + myLock->lock.l_type = F_WRLCK; + myLock->lock.l_whence = 0; + myLock->lock.l_start = 0; + myLock->lock.l_len = strlen(tmp)*sizeof(char); + while (true) { + if((res=fcntl(fileno(myLock->lockfile), F_SETLK, &(myLock->lock)))==-1 && + (errno==EAGAIN || errno==EACCES)){ + if(cnt >= ZOO_LOCK_MAX_RETRY){ + char message[51]; + sprintf(message,"Unable to get the lock after %d attempts.\n",cnt); + setMapInMaps(conf,"lenv","message",message); + fclose(myLock->lockfile); + free(myLock->filename); + free(myLock); + return NULL; + } + fprintf(stderr,"(%d) Wait for lock on %s, tried %d times ... \n",getpid(),myLock->filename,cnt); + fflush(stderr); + sleep(1); + cnt++; + }else + break; + } + if(res<0){ + char *tmp; + if(errno==EBADF) + tmp="Either: the filedes argument is invalid; you requested a read lock but the filedes is not open for read access; or, you requested a write lock but the filedes is not open for write access."; + else + if(errno==EINVAL) + tmp="Either the lockp argument doesn’t specify valid lock information, or the file associated with filedes doesn’t support locks."; + else + tmp="The system has run out of file lock resources; there are already too many file locks in place."; + fprintf(stderr,"Unable to get the lock on %s due to the following error: %s\n",myLock->filename,tmp); + return NULL; + } + return myLock; + } +} + +/** + * Remove a lock. + * @param conf the main configuration maps + * @param s the zooLock structure + * @return 0 on success, -1 on failure. + */ +int unlockFile(maps* conf,struct zooLock* s){ + int res=-1; + if(s!=NULL){ + s->lock.l_type = F_UNLCK; + res=fcntl(fileno(s->lockfile), F_SETLK, &s->lock); + if(res==-1) + return res; + // Check if there is any process locking a file and delete the lock if not. + s->lock.l_type = F_WRLCK; + if(fcntl(fileno(s->lockfile), F_GETLK, &s->lock)!=-1 && s->lock.l_type == F_UNLCK){ + unlink(s->filename); + } + fclose(s->lockfile); + free(s->filename); + free(s); + } + return res; +} + #ifndef RELY_ON_DB #include Index: branches/prototype-v0/zoo-project/zoo-kernel/service_internal.h =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/service_internal.h (revision 851) +++ branches/prototype-v0/zoo-project/zoo-kernel/service_internal.h (revision 854) @@ -63,5 +63,5 @@ * Number of time the ZOO-Kernel will try to acquire lock */ -#define ZOO_LOCK_MAX_RETRY 10 +#define ZOO_LOCK_MAX_RETRY 180 #include @@ -85,4 +85,6 @@ #endif +#include + #include "service.h" @@ -94,11 +96,28 @@ #endif +/** + * The lock structure used by the ZOO-Kernel to ensure atomicity of operations + * + */ +typedef struct zooLock{ + struct flock lock; //!< The lock + FILE* lockfile; //!< The pointer to the lock file + char* filename; //!< The filename to lock +} zooLock; + +static zooLock** zoo_file_locks=NULL; +static int zoo_file_locks_cnt=0; + #ifdef __cplusplus extern "C" { #endif + ZOO_DLL_EXPORT char *readVSIFile(maps*,const char*); ZOO_DLL_EXPORT int setOutputValue( maps*, const char*, char*, size_t); ZOO_DLL_EXPORT char* getInputValue( maps*,const char*,size_t*); + + ZOO_DLL_EXPORT struct zooLock* lockFile(maps*,const char*,const char); + ZOO_DLL_EXPORT int unlockFile(maps*,struct zooLock*); ZOO_DLL_EXPORT void unhandleStatus(maps*); Index: branches/prototype-v0/zoo-project/zoo-kernel/service_internal_hpc.c =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/service_internal_hpc.c (revision 851) +++ branches/prototype-v0/zoo-project/zoo-kernel/service_internal_hpc.c (revision 854) @@ -144,8 +144,10 @@ if(cur->defaults!=NULL){ freeIOType(&cur->defaults); + free(cur->defaults); cur->defaults=NULL; } if(cur->supported!=NULL){ freeIOType(&cur->supported); + free(cur->supported); cur->supported=NULL; } @@ -162,4 +164,115 @@ } //dumpElements((*s)->outputs); +} + +/** + * Acquire a read lock on every files used as input for executing a service. + * @param conf the main configuration file map + * @return 0 if every file can be locked, -1 if one lock has failed. + */ +int addReadLocks(maps** conf){ + map* queueLengthMap=getMapFromMaps(*conf,"uploadQueue","length"); + maps* queueMaps=getMaps(*conf,"uploadQueue"); + if(queueLengthMap!=NULL){ + int cnt=atoi(queueLengthMap->value); + int i=0; + for(i=0;icontent,"input",i), + getMapArray(queueMaps->content,"localPath",i) + }; + zooLock* lck; + if((lck=lockFile(*conf,argv[1]->value,'r'))==NULL){ + char* templateStr=_("Unable to lock the file for %s in read mode."); + char *tmpMessage=(char*)malloc((strlen(templateStr)+strlen(argv[0]->value)+1)*sizeof(char)); + sprintf(tmpMessage,templateStr,argv[0]->value); + setMapInMaps(*conf,"lenv","message",tmpMessage); + free(tmpMessage); + return -1; + }else{ + if(zoo_file_locks_cnt==0){ + zoo_file_locks=(zooLock**)malloc(sizeof(zooLock*)); + } + else{ + zoo_file_locks=(zooLock**)realloc(zoo_file_locks,(zoo_file_locks_cnt+1)*sizeof(zooLock*)); + } + zoo_file_locks[zoo_file_locks_cnt]=lck; + zoo_file_locks_cnt++; + } + } + } + return 0; +} + +/** + * Remove all read locks set for files used as input for executing the service. + * @param conf the main configuration maps pointer + * @return 0 in case of success, -1 if any error occured. In case of error, one + * can refer to the message map array from the lenv section. + */ +int removeReadLocks(maps** conf){ + int res=0; + int nberr=0; + map* queueLengthMap=getMapFromMaps(*conf,"uploadQueue","length"); + maps* queueMaps=getMaps(*conf,"uploadQueue"); + if(queueLengthMap!=NULL){ + int cnt=atoi(queueLengthMap->value); + int i=0; + for(i=0;icontent,"input",i); + char* templateStr=_("Unable to unlock the file for %s after execution."); + char *tmpMessage=(char*)malloc((strlen(templateStr)+strlen(argv->value)+1)*sizeof(char)); + sprintf(tmpMessage,templateStr,argv->value); + maps* lenv=getMaps(*conf,"lenv"); + setMapArray(lenv->content,"message",nberr,tmpMessage); + free(tmpMessage); + res=-1; + nberr++; + } + } + } + free(zoo_file_locks); + return res; +} + +/** + * Get the section name depending on number of features and/or pixels of each + * inputs and the threshold defined in a section. + * It supposes that your inputs has been published using MapServer support, + * implying that the number of features (nb_features), respectively pixels + * (nb_pixels), are defined. The section, identified by confId, should contain + * preview_max_features and preview_max_pixels defining the threshold values. + * @param conf the main configuration file maps pointer + * @param inputs the inputs maps pointer + * @param confId the section identifier + * @return "preview_conf" in case the numbers are lower than the threshold, + * "fullres_conf" in other cases. + */ +char* getConfiguration(maps** conf,maps** inputs,const char* confId){ + maps* input=*inputs; + map* max_pixels=getMapFromMaps(*conf,confId,"preview_max_pixels"); + map* max_features=getMapFromMaps(*conf,confId,"preview_max_features"); + int i_max_pixels=atoi(max_pixels->value); + int i_max_features=atoi(max_features->value); + while(input!=NULL && input->content!=NULL){ + map* tmpMap=getMap(input->content,"geodatatype"); + if(tmpMap!=NULL){ + map* currentNb; + if(strcasecmp(tmpMap->value,"raster")==0 ){ + currentNb=getMap(input->content,"nb_pixels"); + if(atoi(currentNb->value)>i_max_pixels) + return "fullres_conf"; + }else{ + if(strcasecmp(tmpMap->value,"vector")==0 ){ + currentNb=getMap(input->content,"nb_features"); + if(atoi(currentNb->value)>i_max_features) + return "fullres_conf"; + } + } + } + input=input->next; + } + return "preview_conf"; } @@ -181,4 +294,6 @@ map* tmp=NULL; int res=-1; + // Get the configuration id depending on service type and defined thresholds + // then, set the configId key in the lenv section char *serviceType; map* mServiceType=getMap(s->content,"serviceType"); @@ -187,7 +302,21 @@ else serviceType="HPC"; - map* targetPathMap=getMapFromMaps(*main_conf,serviceType,"storagePath"); map* tmpPath=getMapFromMaps(*main_conf,"main","tmpPath"); map* uuid=getMapFromMaps(*main_conf,"lenv","usid"); + map* confMap=getMapFromMaps(*main_conf,serviceType,getConfiguration(main_conf,real_inputs,serviceType)); + char * configurationId=confMap->value; + setMapInMaps(*main_conf,"lenv","configId",configurationId); + // Dump lenv maps again after having set the configId ... + char *flenv = + (char *) + malloc ((strlen (tmpPath->value) + + strlen (uuid->value) + 12) * sizeof (char)); + sprintf (flenv, "%s/%s_lenv.cfg", tmpPath->value, uuid->value); + maps* lenvMaps=getMaps(m,"lenv"); + dumpMapsToFile(lenvMaps,flenv,0); + free(flenv); + + map* targetPathMap=getMapFromMaps(*main_conf,configurationId,"remote_data_path"); + pthread_t threads_pool[50]; // Force the HPC services to be called asynchronously @@ -215,4 +344,5 @@ dumpMapsValuesToFiles(main_conf,&input); addToMap(input->content,"toPublish","true"); + addToMap(input->content,"useMapserver","true"); } if(getMap(input->content,"cache_file")!=NULL){ @@ -264,24 +394,31 @@ invokeCallback(m,inputs,NULL,2,0); fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + dumpMaps(inputs); + if(getMapFromMaps(m,"lenv","mapError")!=NULL){ + invokeCallback(m,inputs,NULL,7,0); + return -1; + } // Upload data on HPC - runUpload(main_conf); - + if(runUpload(main_conf)==false){ + errorException (m, _("Unable to lock the file for upload!"), + "InternalError", NULL); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + invokeCallback(m,inputs,NULL,7,0); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + return -1; + } fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); invokeCallback(m,inputs,NULL,2,1); fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); - // Add the filename to generate for every output to the parameters + // Add the filename to generate for every output to parameters input=*real_outputs; // TODO: fix appendOutputParameters //appendOutputParameters(input,parameters,¶meters_cnt,s,uuid,targetPathMap); + dumpMaps(input); while(input!=NULL){ // TODO: parse all outputs including inner outputs if required. if(input->child==NULL){ - parameters_cnt+=1; - if(parameters_cnt==1) - parameters=(char**)malloc(parameters_cnt*sizeof(char*)); - else - parameters=(char**)realloc(parameters,parameters_cnt*sizeof(char*)); // Name every files that should be produced by the service execution map* mime=getMap(input->content,"mimeType"); @@ -302,23 +439,27 @@ free(targetName); setMapInMaps(*real_outputs,input->name,"generated_file",targetPath); - // We should verify if any optional tag for output is required - // (i.e. -out output.tiff *int8*), meaning that we should search - // for a corresponding inputs name. - map* inValue=getMapFromMaps(*real_inputs,input->name,"value"); - if(inValue!=NULL){ - parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(targetPath)+strlen(inValue->value)+4)*sizeof(char)); - sprintf(parameters[parameters_cnt-1],"-%s %s %s",input->name,targetPath,inValue->value); - }else{ - parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(targetPath)+3)*sizeof(char)); - sprintf(parameters[parameters_cnt-1],"-%s %s",input->name,targetPath); + if(strcasecmp(input->name,"wms_link")!=0&& + strcasecmp(input->name,"wcs_link")!=0 && + strcasecmp(input->name,"wfs_link")!=0){ + parameters_cnt+=1; + if(parameters_cnt==1) + parameters=(char**)malloc(parameters_cnt*sizeof(char*)); + else + parameters=(char**)realloc(parameters,parameters_cnt*sizeof(char*)); + // We should verify if any optional tag for output is required + // (i.e. -out output.tiff *int8*), meaning that we should search + // for a corresponding inputs name. + map* inValue=getMapFromMaps(*real_inputs,input->name,"value"); + if(inValue!=NULL){ + parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(targetPath)+strlen(inValue->value)+4)*sizeof(char)); + sprintf(parameters[parameters_cnt-1],"-%s %s %s",input->name,targetPath,inValue->value); + }else{ + parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(targetPath)+3)*sizeof(char)); + sprintf(parameters[parameters_cnt-1],"-%s %s",input->name,targetPath); + } } free(targetPath); }// In other case it means we need to return the cache_file as generated_file else{ - parameters_cnt+=1; - if(parameters_cnt==1) - parameters=(char**)malloc(parameters_cnt*sizeof(char*)); - else - parameters=(char**)realloc(parameters,parameters_cnt*sizeof(char*)); // Name every files that should be produced by the service execution map* mime=getMap(input->child->content,"mimeType"); @@ -339,14 +480,24 @@ free(targetName); addToMap(input->content,"generated_file",targetPath); - // We should verify if any optional tag for output is required - // (i.e. -out output.tiff *int8*), meaning that we should search - // for a corresponding inputs name. - map* inValue=getMapFromMaps(*real_inputs,input->name,"value"); - if(inValue!=NULL){ - parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(targetPath)+strlen(inValue->value)+4)*sizeof(char)); - sprintf(parameters[parameters_cnt-1],"-%s %s %s",input->name,targetPath,inValue->value); - }else{ - parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(targetPath)+3)*sizeof(char)); - sprintf(parameters[parameters_cnt-1],"-%s %s",input->name,targetPath); + addToMap(input->content,"storage",targetPath); + if(strcasecmp(input->name,"wms_link")!=0&& + strcasecmp(input->name,"wcs_link")!=0 && + strcasecmp(input->name,"wfs_link")!=0){ + parameters_cnt+=1; + if(parameters_cnt==1) + parameters=(char**)malloc(parameters_cnt*sizeof(char*)); + else + parameters=(char**)realloc(parameters,parameters_cnt*sizeof(char*)); + // We should verify if any optional tag for output is required + // (i.e. -out output.tiff *int8*), meaning that we should search + // for a corresponding inputs name. + map* inValue=getMapFromMaps(*real_inputs,input->name,"value"); + if(inValue!=NULL){ + parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(targetPath)+strlen(inValue->value)+4)*sizeof(char)); + sprintf(parameters[parameters_cnt-1],"-%s %s %s",input->name,targetPath,inValue->value); + }else{ + parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(targetPath)+3)*sizeof(char)); + sprintf(parameters[parameters_cnt-1],"-%s %s",input->name,targetPath); + } } free(targetPath); @@ -354,5 +505,4 @@ input=input->next; } - // Produce the SBATCH File locally char *scriptPath=(char*)malloc((strlen(s->name)+strlen(tmpPath->value)+strlen(uuid->value)+10)*sizeof(char)); @@ -365,5 +515,5 @@ fflush(stderr); FILE* scriptFile=fopen(scriptPath,"w+"); - map* headerMap=getMapFromMaps(*main_conf,serviceType,"header"); + map* headerMap=getMapFromMaps(*main_conf,configurationId,"jobscript_header"); if(headerMap!=NULL){ // Use the header file if defined in the HPC section of the main.cfg file @@ -383,9 +533,10 @@ }else fprintf(scriptFile,"#!/bin/bash\n\n### *** Default ZOO-Service HEADER *** ###\n\n"); - maps* hpc_opts=getMaps(*main_conf,"sbatch_options"); + maps* hpc_opts=getMaps(*main_conf,configurationId); if(hpc_opts!=NULL){ map* hpc_opts_content=hpc_opts->content; while(hpc_opts_content!=NULL){ - fprintf(scriptFile,"#SBATCH --%s=%s\n",hpc_opts_content->name,hpc_opts_content->value); + if(strncasecmp(hpc_opts_content->name,"sbatch_options_",15)==0) + fprintf(scriptFile,"#SBATCH --%s=%s\n",strstr(hpc_opts_content->name,"sbatch_options_")+15,hpc_opts_content->value); hpc_opts_content=hpc_opts_content->next; } @@ -396,5 +547,5 @@ fprintf(scriptFile,"#SBATCH --export=MODULES=%s\n",mods->value); - map* bodyMap=getMapFromMaps(*main_conf,serviceType,"body"); + map* bodyMap=getMapFromMaps(*main_conf,configurationId,"jobscript_body"); if(bodyMap!=NULL){ // Use the header file if defined in the HPC section of the main.cfg file @@ -415,5 +566,4 @@ fprintf(scriptFile,"#!/bin/bash\n\n### *** Default ZOO-Service BODY *** ###\n\n"); - map* sp=getMap(s->content,"serviceProvider"); @@ -433,5 +583,4 @@ fflush(scriptFile); fclose(scriptFile); - fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); invokeCallback(m,inputs,NULL,3,1); @@ -442,7 +591,15 @@ invokeCallback(m,inputs,NULL,4,0); fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); - targetPathMap=getMapFromMaps(*main_conf,serviceType,"executePath"); + targetPathMap=getMapFromMaps(*main_conf,configurationId,"remote_work_path"); if(targetPathMap==NULL){ - setMapInMaps(*main_conf,"lenv","message",_("There is no executePath defined in you HPC section!")); + setMapInMaps(*main_conf,"lenv","message",_("There is no remote_work_path defined in your section!")); + setMapInMaps(*main_conf,"lenv","status","failed"); + errorException (m, _("There is no remote_work_path defined in your section!"), + "InternalError", NULL); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); + invokeCallback(m,NULL,NULL,7,0); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); return SERVICE_FAILED; } @@ -453,7 +610,9 @@ SSHCON *test=ssh_connect(*main_conf); ssh_copy(*main_conf,scriptPath,targetPath,ssh_get_cnt(*main_conf)); - + unlink(scriptPath); + free(scriptPath); // Execute the SBATCH script remotely - map* subStr=getMapFromMaps(*main_conf,"HPC","subStr"); + addReadLocks(main_conf); + map* subStr=getMapFromMaps(*main_conf,configurationId,"sbatch_substr"); char *command=(char*)malloc((strlen(targetPath)+strlen(targetPathMap->value)+strlen(subStr->value)+strlen(uuid->value)+137)*sizeof(char)); sprintf(command,"sbatch %s 2> %s/error_%s.log | sed \"s:%s::g\"",targetPath,targetPathMap->value,uuid->value,subStr->value); @@ -489,5 +648,5 @@ fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); fflush(stderr); - invokeCallback(m,NULL,NULL,7,1); + invokeCallback(m,NULL,NULL,7,0); fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); fflush(stderr); @@ -497,5 +656,5 @@ free(targetPath); ssh_close(*main_conf); - sleep(120); + sleep(1); return -1; } @@ -507,26 +666,70 @@ fflush(stderr); free(command); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); struct sockaddr_un addr; + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); memset(&addr, 0, sizeof(addr)); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); addr.sun_family = AF_UNIX; + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); int rc, cl, fd = socket(AF_UNIX, SOCK_STREAM, 0); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); char *sname=(char*)malloc((strlen(tmpPath->value)+strlen(uuid->value)+20)); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); sprintf(sname,"%s/.wait_socket_%s.sock",tmpPath->value,uuid->value); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); strncpy(addr.sun_path, sname, sizeof(addr.sun_path)-1); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); if (bind(fd, (struct sockaddr*)&addr, sizeof(addr)) == -1) { perror("bind error"); + setMapInMaps(m,"lenv","message",_("Unable to bind socket!")); + errorException (m, _("Unable to bind socket!"), + "InternalError", NULL); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); + invokeCallback(m,NULL,NULL,7,0); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); sleep(120); return -1; } + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); if (listen(fd, 5) == -1) { setMapInMaps(*main_conf,"lenv","message",_("Listen error")); + errorException (m, _("Listen error"), + "InternalError", NULL); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); + invokeCallback(m,NULL,NULL,7,0); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); return -1; } + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); if ( (cl = accept(fd, NULL, NULL)) == -1) { setMapInMaps(*main_conf,"lenv","message",_("Accept error")); + errorException (m, _("Accept error"), + "InternalError", NULL); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); + invokeCallback(m,NULL,NULL,7,0); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); return -1; }else{ + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); int hasPassed=-1; char buf[11]; @@ -536,18 +739,41 @@ sleep(1); setMapInMaps(*main_conf,"lenv","message",_("Read closed")); - invokeCallback(m,NULL,NULL,7,1); + errorException (m, _("Read closed"), + "InternalError", NULL); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); + invokeCallback(m,NULL,NULL,7,0); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); return -1; }else{ if(rc<0){ setMapInMaps(*main_conf,"lenv","message",_("Read error")); - invokeCallback(m,NULL,NULL,7,1); + errorException (m, _("Read error"), + "InternalError", NULL); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); + invokeCallback(m,NULL,NULL,7,0); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); return -1; } } hasPassed=1; + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); res=atoi(buf); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); unlink(sname); - //free(sname); - + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); + free(sname); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); + removeReadLocks(main_conf); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); + if(res==3){ fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); @@ -573,4 +799,5 @@ setMapInMaps(*main_conf,"lenv","message",tmpStr); free(tmpStr); + invokeCallback(m,NULL,NULL,7,0); return SERVICE_FAILED; } @@ -587,4 +814,5 @@ if(ssh_fetch(*main_conf,targetPath,generatedFile->value,ssh_get_cnt(m))==0){ maps* tmp=getMaps(*real_outputs,input->name); + char serviceName[9]; freeMap(&tmp->content); free(tmp->content); @@ -592,11 +820,34 @@ maps* output=getMaps(*real_outputs,input->name); setMapInMaps(output->child,"download_link","generated_file",targetPath); + setMapInMaps(output->child,"download_link","storage",targetPath); setMapInMaps(output->child,"download_link","useMapserver","false"); + setMapInMaps(output->child,"download_link","replicateStorageNext","true"); + setMapInMaps(output->child,"download_link","asReference","true"); + setMapInMaps(output->child,"download_link","inRequest","true"); setMapInMaps(output->child,"wms_link","generated_file",targetPath); + setMapInMaps(output->child,"wms_link","storage",targetPath); setMapInMaps(output->child,"wms_link","useMapserver","true"); setMapInMaps(output->child,"wms_link","msOgc","WMS"); setMapInMaps(output->child,"wms_link","requestedMimeType","image/png"); - setMapInMaps(output->child,"wcs_link","generated_file",targetPath); - setMapInMaps(output->child,"wcs_link","useMapserver","true"); + setMapInMaps(output->child,"wms_link","asReference","true"); + if(getMaps(output->child,"wcs_link")!=NULL){ + sprintf(serviceName,"wcs_link"); + setMapInMaps(output->child,serviceName,"msOgc","WCS"); + } + else{ + sprintf(serviceName,"wfs_link"); + setMapInMaps(output->child,serviceName,"msOgc","WFS"); + } + setMapInMaps(output->child,serviceName,"storage",targetPath); + setMapInMaps(output->child,serviceName,"generated_file",targetPath); + setMapInMaps(output->child,serviceName,"useMapserver","true"); + setMapInMaps(output->child,serviceName,"asReference","true"); + }else{ + char *tmpStr=(char*)malloc((strlen(filename)+strlen(_("Unable to fetch the remote file for %s"))+1)*sizeof(char)); + sprintf(tmpStr,_("Unable to fetch the remote file for %s"),filename); + setMapInMaps(*main_conf,"lenv","message",tmpStr); + free(tmpStr); + invokeCallback(m,NULL,NULL,7,0); + return SERVICE_FAILED; } free(targetPath); @@ -605,4 +856,35 @@ input=input->next; } + + // Read informations provided by FinalizeHPC as a configuration file + // then, remove the file. + map* jobid=getMapFromMaps(*main_conf,"lenv","usid"); + map* tmpPath=getMapFromMaps(*main_conf,"main","tmpPath"); + char *filePath=(char*)malloc((strlen(tmpPath->value)+strlen(jobid->value)+15)*sizeof(char)); + sprintf(filePath,"%s/exec_status_%s",tmpPath->value,jobid->value); + maps* m = (maps *) malloc (MAPS_SIZE); + m->child=NULL; + m->next=NULL; + int saved_stdout = dup (fileno (stdout)); + dup2 (fileno (stderr), fileno (stdout)); + conf_read(filePath,m); + fflush(stdout); + dup2 (saved_stdout, fileno (stdout)); + close(saved_stdout); + unlink(filePath); + free(filePath); + addMapsToMaps(main_conf,m); + freeMaps(&m); + free(m); + }else{ + // Try to access remotely to the log file and return a more relevant error message + setMapInMaps(m,"lenv","message",_("HPC Execution failed!")); + errorException (m, _("HPC Execution failed!"), + "InternalError", NULL); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); + invokeCallback(m,NULL,NULL,7,0); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); } //free(buf); @@ -611,4 +893,11 @@ perror("Failed to read"); setMapInMaps(*main_conf,"lenv","message",_("Unable to parse the value returned by remote execution")); + errorException (m, _("Unable to parse the value returned by remote execution"), + "InternalError", NULL); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); + invokeCallback(m,NULL,NULL,7,0); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + fflush(stderr); sleep(120); return SERVICE_FAILED; @@ -622,4 +911,2 @@ return res; } - - Index: branches/prototype-v0/zoo-project/zoo-kernel/service_internal_ms.c =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/service_internal_ms.c (revision 851) +++ branches/prototype-v0/zoo-project/zoo-kernel/service_internal_ms.c (revision 854) @@ -138,4 +138,5 @@ exit(-1); } + int finalProto=-1; map *msOgcVersion=getMapFromMaps(m,"main","msOgcVersion"); map *dataPath=getMapFromMaps(m,"main","dataPath"); @@ -147,10 +148,19 @@ map* protoMap=getMap(tmpI->content,"msOgc"); map* versionMap=getMap(tmpI->content,"msOgcVersion"); - char options[3][5][25]={ + char options[4][5][25]={ {"WMS","1.3.0","GetMap","layers=%s","wms_extent"}, {"WFS","1.1.0","GetFeature","typename=%s","wcs_extent"}, - //{"WCS","1.1.0","GetCoverage","coverage=%s","wcs_extent"} - {"WCS","2.0.0","GetCoverage","coverageid=%s","wcs_extent"} + {"WCS","2.0.0","GetCoverage","coverageid=%s","wcs_extent"}, + {"WCS","1.1.0","GetCoverage","coverage=%s","wcs_extent"} }; + map* datatype=getMap(tmpI->content,"geodatatype"); + if(datatype==NULL || strncmp(datatype->value,"other",5)==0){ + setMapInMaps(m,"lenv","mapError","true"); + setMapInMaps(m,"lenv","locator",tmpI->name); + setMapInMaps(m,"lenv","message",_("The ZOO-Kernel was able to retrieve the data but could not read it as geographic data.")); + if(getMapFromMaps(m,"lenv","state")==NULL) + errorException (m, _("Unable to find any geographic data"), "WrongInputData", tmpI->name); + return; + } int proto=0; if(rformat==NULL){ @@ -194,7 +204,8 @@ char* webService_url=(char*)malloc((strlen(msUrl->value)+strlen(format->value)+strlen(tmpI->name)+strlen(width->value)+strlen(height->value)+strlen(extent->value)+256)*sizeof(char)); - map* datatype=getMap(tmpI->content,"geodatatype"); if(proto>0){ + if(proto==2) + finalProto=1; sprintf(webService_url, "%s?map=%s/%s_%s.map&request=%s&service=%s&version=%s&%s&format=%s&bbox=%s&crs=%s", @@ -248,9 +259,12 @@ } addToMap(tmpI->content,"Reference",webService_url); - + memset(layers,0,128); + sprintf(layers,options[proto][3],tmpI->name); protoVersion=options[proto][1]; extent=getMap(tmpI->content,options[proto][4]); memset(webService_url,0,strlen(webService_url)); if(proto>0){ + if(proto==2) + finalProto=1; sprintf(webService_url, "%s?map=%s/%s_%s.map&request=%s&service=%s&version=%s&%s&format=%s&bbox=%s&crs=%s", @@ -292,4 +306,27 @@ addToMap(tmpI->content,"ref_wms_link",webService_url); } + if(finalProto>0){ + proto=3; + memset(layers,0,128); + sprintf(layers,options[proto][3],tmpI->name); + protoVersion=options[proto][1]; + extent=getMap(tmpI->content,options[proto][4]); + memset(webService_url,0,strlen(webService_url)); + sprintf(webService_url, + "%s?map=%s/%s_%s.map&request=%s&service=%s&version=%s&%s&format=%s&bbox=%s&crs=%s", + msUrl->value, + dataPath->value, + tmpI->name, + sid->value, + options[proto][2], + options[proto][0], + protoVersion, + layers, + rformat->value, + extent->value, + crs->value + ); + addToMap(tmpI->content,"ref_wcs_preview_link",webService_url); + } if(hasCRS==0){ freeMap(&crs); @@ -299,5 +336,4 @@ free(rformat); free(webService_url); - dumpMaps(tmpI); } @@ -492,6 +528,5 @@ /** * Try to load the file as ZIP - */ - + * OGRDataSourceH poDS1 = NULL; OGRSFDriverH *poDriver1 = NULL; @@ -534,4 +569,6 @@ fprintf(stderr,"Unable to access the DataSource as ZIP File\n"); setMapInMaps(conf,"lenv","message","Unable to open datasource in read only mode"); + fprintf(stderr,"Remove ZIP File!\n"); + unlink(odsName); //OGR_DS_Destroy(poDS1); }else{ @@ -594,7 +631,7 @@ } OGR_DS_Destroy(poDS1); - } + } free(sdsName); - free(dsName); + free(dsName);*/ OGRDataSourceH poDS = NULL; @@ -649,4 +686,6 @@ myLayer->status = MS_ON; msConnectLayer(myLayer,MS_OGR,pszDataSource); + + addIntToMap(output->content,"nb_features",OGR_L_GetFeatureCount(poLayer,1)); /** @@ -811,5 +850,5 @@ */ GDALAllRegister(); - hDataset = GDALOpen( pszFilename, GA_ReadOnly ); + hDataset = GDALOpen( pszFilename, GA_Update ); if( hDataset == NULL ){ #ifdef DEBUGMS @@ -866,4 +905,5 @@ m->width=GDALGetRasterXSize( hDataset ); m->height=GDALGetRasterYSize( hDataset ); + addIntToMap(output->content,"nb_pixels",GDALGetRasterXSize( hDataset )*GDALGetRasterYSize( hDataset )); /** @@ -880,5 +920,9 @@ setSrsInformations(output,m,myLayer,pszProjection); }else{ - fprintf(stderr,"NO SRS FOUND ! %s\n",GDALGetProjectionRef( hDataset )); + fprintf(stderr,"NO SRS FOUND ! %s\n",GDALGetProjectionRef( hDataset )); + CPLErr sp=GDALSetProjection( hDataset , "+init=epsg:4326" ); + if(sp!=CE_None){ + fprintf(stderr,"NO SRS FOUND ! %s\n",CPLGetLastErrorMsg()); + } } @@ -905,4 +949,12 @@ addToMap(output->content,"boundingbox",extent); } + }else{ + int scale=1; + if(m->width>2048){ + addIntToMap(output->content,"width",2048); + scale=2048/m->width; + }else + addIntToMap(output->content,"width",m->width); + addIntToMap(output->content,"height",m->height*scale); } @@ -919,9 +971,13 @@ } if(nBandsI>=3) - msLayerAddProcessing(myLayer,"BANDS=1,2,3"); + if(nBandsI==4) + msLayerAddProcessing(myLayer,"BANDS=1,2,3,4"); + else + msLayerAddProcessing(myLayer,"BANDS=1,2,3"); else if(nBandsI>=2) msLayerAddProcessing(myLayer,"BANDS=1,2"); else msLayerAddProcessing(myLayer,"BANDS=1"); + /** @@ -929,7 +985,7 @@ */ char lBands[7]; - memset(&nBands,0,7); char *nameBands=NULL; for( iBand = 0; iBand < nBandsI; iBand++ ){ + memset(&lBands,0,7); sprintf(lBands,"Band%d",iBand+1); if(nameBands==NULL){ @@ -937,5 +993,5 @@ sprintf(nameBands,"%s",lBands); }else{ - if(iBand<4){ + /*if(iBand<4)*/{ char *tmpS=zStrdup(nameBands); nameBands=(char*)realloc(nameBands,(strlen(tmpS)+strlen(lBands)+2)*sizeof(char)); @@ -954,21 +1010,20 @@ */ for( iBand = 0; iBand < nBandsI; iBand++ ){ - //int bGotNodata;//, bSuccess; - double adfCMinMax[2]/*, dfNoData*/; - //int nBlockXSize, nBlockYSize, nMaskFlags; - //double /*dfMean, dfStdDev*/; + double pdfMin, pdfMax, pdfMean, pdfStdDev; hBand = GDALGetRasterBand( hDataset, iBand+1 ); CPLErrorReset(); - GDALComputeRasterMinMax( hBand, FALSE, adfCMinMax ); + GDALComputeRasterStatistics( hBand, TRUE, &pdfMin, &pdfMax, &pdfMean, &pdfStdDev, NULL,NULL); char tmpN[21]; sprintf(tmpN,"Band%d",iBand+1); if (CPLGetLastErrorType() == CE_None){ - char tmpMm[100]; - sprintf(tmpMm,"%.3f %.3f",adfCMinMax[0],adfCMinMax[1]); + char tmpMm[100],tmpMp[100]; + sprintf(tmpMm,"%.3f %.3f",pdfMin,pdfMax); + sprintf(tmpMp,"SCALE_%d=%.3f,%.3f",iBand+1,pdfMean-(2*pdfStdDev),pdfMean+(2*pdfStdDev)); char tmpI[31]; sprintf(tmpI,"%s_interval",tmpN); msInsertHashTable(&(myLayer->metadata), tmpI, tmpMm); - + if(pdfMax>255) + msLayerAddProcessing(myLayer,tmpMp); map* test=getMap(output->content,"msClassify"); if(test!=NULL && strncasecmp(test->value,"true",4)==0){ @@ -990,7 +1045,7 @@ if(nBandsI==1){ - double delta=adfCMinMax[1]-adfCMinMax[0]; + double delta=pdfMax-pdfMin; double interval=delta/10; - double cstep=adfCMinMax[0]; + double cstep=pdfMin; for(i=0;i<10;i++){ /** @@ -1036,5 +1091,5 @@ char tmpMm[100]; - sprintf(tmpMm,"%.3f %.3f",adfCMinMax[0],adfCMinMax[1]); + sprintf(tmpMm,"%.3f %.3f",pdfMin,pdfMax); } @@ -1046,5 +1101,4 @@ myLayer->offsite.blue=0; } - msLayerAddProcessing(myLayer,"RESAMPLE=BILINEAR"); } } @@ -1056,4 +1110,5 @@ } + msLayerAddProcessing(myLayer,"RESAMPLE=BILINEAR"); m->layerorder[m->numlayers] = m->numlayers; @@ -1081,24 +1136,27 @@ ext="json"; - map* tmpMap=getMapFromMaps(conf,"main","dataPath"); - map* sidMap=getMapFromMaps(conf,"lenv","usid"); - char *pszDataSource=(char*)malloc((strlen(tmpMap->value)+strlen(sidMap->value)+strlen(outputs->name)+17)*sizeof(char)); - sprintf(pszDataSource,"%s/ZOO_DATA_%s_%s.%s",tmpMap->value,outputs->name,sidMap->value,ext); - int f=zOpen(pszDataSource,O_WRONLY|O_CREAT,S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH); - map *gfile=getMap(outputs->content,"generated_file"); - if(gfile!=NULL){ - readGeneratedFile(conf,outputs->content,gfile->value); - } - map* sizeMap=getMap(outputs->content,"size"); - map* vData=getMap(outputs->content,"value"); - if(sizeMap!=NULL){ - zWrite(f,vData->value,atoi(sizeMap->value)*sizeof(char)); - } - else{ - zWrite(f,vData->value,(strlen(vData->value)+1)*sizeof(char)); - } - close(f); - addToMap(outputs->content,"storage",pszDataSource); - free(pszDataSource); + map* storage=getMap(outputs->content,"storage"); + if(storage==NULL){ + map* tmpMap=getMapFromMaps(conf,"main","dataPath"); + map* sidMap=getMapFromMaps(conf,"lenv","usid"); + char *pszDataSource=(char*)malloc((strlen(tmpMap->value)+strlen(sidMap->value)+strlen(outputs->name)+17)*sizeof(char)); + sprintf(pszDataSource,"%s/ZOO_DATA_%s_%s.%s",tmpMap->value,outputs->name,sidMap->value,ext); + int f=zOpen(pszDataSource,O_WRONLY|O_CREAT,S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH); + map *gfile=getMap(outputs->content,"generated_file"); + if(gfile!=NULL){ + readGeneratedFile(conf,outputs->content,gfile->value); + } + map* sizeMap=getMap(outputs->content,"size"); + map* vData=getMap(outputs->content,"value"); + if(sizeMap!=NULL){ + zWrite(f,vData->value,atoi(sizeMap->value)*sizeof(char)); + } + else{ + zWrite(f,vData->value,(strlen(vData->value)+1)*sizeof(char)); + } + close(f); + addToMap(outputs->content,"storage",pszDataSource); + free(pszDataSource); + } /* @@ -1174,4 +1232,15 @@ #endif + outputFormatObj *o6=msCreateDefaultOutputFormat(NULL,"GDAL/GTiff","geotiff"); + if(!o6) + fprintf(stderr,"Unable to initialize GDAL driver !\n"); + else{ + o6->imagemode=MS_IMAGEMODE_BYTE; + o6->mimetype=strdup("image/geotiff"); + o6->inmapfile=MS_TRUE; + msAppendOutputFormat(myMap,msCloneOutputFormat(o6)); + msFreeOutputFormat(o6); + } + /* * Set default projection to EPSG:4326 @@ -1262,6 +1331,8 @@ msSaveMap(myMap,mapPath); free(mapPath); + //msFreeSymbolSet(&myMap->symbolset); + msFreeMap(myMap); + //msFree(myMap); msGDALCleanup(); - msFreeMap(myMap); } Index: branches/prototype-v0/zoo-project/zoo-kernel/service_internal_python.c =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/service_internal_python.c (revision 851) +++ branches/prototype-v0/zoo-project/zoo-kernel/service_internal_python.c (revision 854) @@ -57,5 +57,5 @@ * from the ZOO-Kernel Python environment: * - "_" corresponding to the PythonTranslate function - * - "updte_status" corresponding to the PythonUpdateStatus function + * - "update_status" corresponding to the PythonUpdateStatus function * @see PythonTranslate, PythonUpdateStatus */ @@ -163,5 +163,5 @@ fprintf(stderr,"PYTHON SUPPORT (%i)\n",strlen(tmp->value)); #endif - python_path=(char*)malloc((strlen(tmp->value))*sizeof(char)); + python_path=(char*)malloc((strlen(tmp->value)+1)*sizeof(char)); sprintf(python_path,"%s",tmp->value); hasToClean=1; @@ -281,5 +281,5 @@ dumpMaps(*real_outputs); #endif - }else{ + }else{ PythonZooReport(m,tmp->value,0); res=-1; @@ -323,10 +323,10 @@ char *pbt=NULL; if(PyString_Check(trace)){ - pbt=(char*)malloc((7+strlen(PyString_AsString(trace))+1)*sizeof(char)); + pbt=(char*)malloc((8+strlen(PyString_AsString(trace)))*sizeof(char)); sprintf(pbt,"TRACE: %s",PyString_AsString(trace)); } else fprintf(stderr,"EMPTY TRACE ?"); - + trace=NULL; Index: branches/prototype-v0/zoo-project/zoo-kernel/sql/schema.sql =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/sql/schema.sql (revision 851) +++ branches/prototype-v0/zoo-project/zoo-kernel/sql/schema.sql (revision 854) @@ -31,6 +31,6 @@ -- Create a dedicated schema to store all tables -- Uncomment the following 2 lines to activate the schema use --- CREATE SCHEMA zoo; --- SET search_path TO zoo; +-- CREATE SCHEMA process; +-- SET search_path TO process; -------------------------------------------------------------------------------- -- Services table @@ -45,10 +45,6 @@ creation_time timestamp with time zone default now(), end_time timestamp with time zone default NULL, - -- queue_duration , - -- run_duration , progress int, - message TEXT, - -- data_lifetime timestamp with time zone default now() + interval '48 hours', - rps_step int4 default 0 + message TEXT ); -------------------------------------------------------------------------------- @@ -56,5 +52,5 @@ -- Used to store the response provided by a services running asynchronously create table responses ( - uuid text references process(uuid) ON DELETE CASCADE, + uuid text references services(uuid) ON DELETE CASCADE, content text, creation_time timestamp with time zone default now() Index: branches/prototype-v0/zoo-project/zoo-kernel/sql/zoo_collectiondb.sql =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/sql/zoo_collectiondb.sql (revision 851) +++ branches/prototype-v0/zoo-project/zoo-kernel/sql/zoo_collectiondb.sql (revision 854) @@ -314,5 +314,5 @@ abstract, (SELECT service_type FROM CollectionDB.zoo_ServiceTypes WHERE id = (SELECT service_type_id FROM CollectionDB.zoo_DeploymentMetadata WHERE id = (SELECT deployment_metadata_id FROM CollectionDB.PrivateMetadataDeploymentmetadataAssignment WHERE private_metadata_id=(SELECT id FROM CollectionDB.zoo_PrivateMetadata WHERE id = CollectionDB.ows_Process.private_metadata_id)))) as service_type, - (SELECT executable_name as service_provider FROM CollectionDB.zoo_DeploymentMetadata WHERE id = (SELECT deployment_metadata_id FROM CollectionDB.PrivateMetadataDeploymentmetadataAssignment WHERE private_metadata_id=(SELECT id FROM CollectionDB.zoo_PrivateMetadata WHERE id = CollectionDB.ows_Process.private_metadata_id))) as service_provider, + (SELECT executable_name as service_provider FROM CollectionDB.zoo_DeploymentMetadata WHERE id = (SELECT deployment_metadata_id FROM CollectionDB.PrivateMetadataDeploymentmetadataAssignment WHERE private_metadata_id=(SELECT id FROM CollectionDB.zoo_PrivateMetadata WHERE id = CollectionDB.ows_Process.private_metadata_id))) as service_provider, availability FROM CollectionDB.ows_Process Index: branches/prototype-v0/zoo-project/zoo-kernel/sqlapi.c =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/sqlapi.c (revision 851) +++ branches/prototype-v0/zoo-project/zoo-kernel/sqlapi.c (revision 854) @@ -70,6 +70,8 @@ int i=0; maps* cconf=getMaps(conf,key); - if(cconf==NULL) + if(cconf==NULL){ + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); return "-1"; + } int len=0; for(i=0;i<6;i++){ @@ -119,6 +121,10 @@ int _init_sql(maps* conf,const char* key){ char* sqlInitString=_createInitString(conf,key); + fprintf(stderr,"Try to connect to: %s %s !\n",key,sqlInitString); + fflush(stderr); if(strncmp(sqlInitString,"-1",2)==0) return -1; + fprintf(stderr,"Try to connect to: %s !\n",key); + fflush(stderr); OGRSFDriver *poDriver = NULL; OGRRegisterAll(); @@ -195,7 +201,9 @@ */ void close_sql(maps* conf,int cid){ - if( zoo_ResultSet != NULL ) + if( zoo_ResultSet != NULL ){ zoo_DS[cid]->ReleaseResultSet( zoo_ResultSet ); - if(zoo_DS!=NULL){ + zoo_ResultSet=NULL; + } + if(zoo_DS!=NULL && zoo_DS[cid]!=NULL){ #if GDAL_VERSION_MAJOR >= 2 GDALClose(zoo_DS[cid]); @@ -223,5 +231,5 @@ */ OGRLayer *fetchSql(maps* conf,int cid,const char* sqlQuery){ - if(zoo_DS[cid]==NULL) + if(zoo_DS==NULL || zoo_DS[cid]==NULL) return NULL; OGRLayer *res=NULL; @@ -250,4 +258,6 @@ int execSql(maps* conf,int cid,const char* sqlQuery){ int res=-1; + if(zoo_DS == NULL || zoo_DS[cid]==NULL) + return -1; zoo_ResultSet = zoo_DS[cid]->ExecuteSQL( sqlQuery, NULL, NULL); if( zoo_ResultSet != NULL ){ @@ -371,17 +381,13 @@ char *sqlQuery=(char*)malloc((strlen(schema->value)+strlen(msg->value)+strlen(p->value)+strlen(sid->value)+64+1)*sizeof(char)); sprintf(sqlQuery,"UPDATE %s.services set status=$$%s$$,message=$$%s$$ where uuid=$$%s$$;",schema->value,p->value,msg->value,sid->value); - if( zoo_ds_nb== -#ifdef META_DB - 1 -#else - 0 -#endif - ){ + if(zoo_ds_nb==0){ init_sql(conf); zoo_ds_nb++; } - execSql(conf,zoo_ds_nb-1,sqlQuery); - cleanUpResultSet(conf,zoo_ds_nb-1); - free(sqlQuery); + execSql(conf,0,sqlQuery); + cleanUpResultSet(conf,0); + close_sql(conf,0); + free(sqlQuery); + setMapInMaps(conf,"lenv","ds_nb","0"); return 0; } @@ -439,4 +445,7 @@ int zoo_ds_nb=getCurrentId(conf); map *schema=getMapFromMaps(conf,"database","schema"); + OGRFeature *poFeature = NULL; + const char *tmp1=NULL; + int hasRes=-1; char *sqlQuery=(char*)malloc((strlen(schema->value)+strlen(pid)+82+1)*sizeof(char)); sprintf(sqlQuery, @@ -454,7 +463,4 @@ } execSql(conf,zoo_ds_nb-1,sqlQuery); - OGRFeature *poFeature = NULL; - const char *tmp1; - int hasRes=-1; if(zoo_ResultSet!=NULL){ while( (poFeature = zoo_ResultSet->GetNextFeature()) != NULL ){ @@ -472,6 +478,5 @@ if(hasRes<0) tmp1=NULL; - else - cleanUpResultSet(conf,zoo_ds_nb-1); + cleanUpResultSet(conf,zoo_ds_nb-1); free(sqlQuery); return (char*)tmp1; @@ -534,5 +539,5 @@ //close_sql(conf,zoo_ds_nb-1); free(sqlQuery); - end_sql(); + //end_sql(); } @@ -555,5 +560,6 @@ zoo_ds_nb++; } - execSql(conf,zoo_ds_nb-1,sqlQuery); + if(execSql(conf,zoo_ds_nb-1,sqlQuery)<0) + return NULL; OGRFeature *poFeature = NULL; const char *tmp1; Index: branches/prototype-v0/zoo-project/zoo-kernel/sshapi.c =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/sshapi.c (revision 851) +++ branches/prototype-v0/zoo-project/zoo-kernel/sshapi.c (revision 854) @@ -29,4 +29,5 @@ #include "sshapi.h" +#include "service_internal.h" SSHCON *sessions[MAX_PARALLEL_SSH_CON]; @@ -94,15 +95,31 @@ int port=22; - fprintf(stderr,"%s %d\n",__FILE__,__LINE__); - fflush(stderr); - - map* hpc_host=getMapFromMaps(conf,"HPC","host"); - map* hpc_port=getMapFromMaps(conf,"HPC","port"); - map* hpc_user=getMapFromMaps(conf,"HPC","user"); - map* hpc_password=getMapFromMaps(conf,"HPC","password"); - map* hpc_public_key=getMapFromMaps(conf,"HPC","key"); - - fprintf(stderr,"%s %d\n",__FILE__,__LINE__); - fflush(stderr); + map* hpc_config=getMapFromMaps(conf,"lenv","configId"); + + map* hpc_host=getMapFromMaps(conf,hpc_config->value,"ssh_host"); + map* hpc_port=getMapFromMaps(conf,hpc_config->value,"ssh_port"); + map* hpc_user=getMapFromMaps(conf,hpc_config->value,"ssh_user"); + map* hpc_password=getMapFromMaps(conf,hpc_config->value,"ssh_password"); + map* hpc_public_key=getMapFromMaps(conf,hpc_config->value,"ssh_key"); + + char ip[100]; + struct hostent *my_hostent; + struct in_addr **addrs; + + if (hpc_host != NULL) { + // Fetch ip address for the hostname + if ( (my_hostent = gethostbyname( hpc_host->value ) ) == NULL){ + herror("gethostbyname"); + setMapInMaps(conf,"lenv","message",_("Issue when invoking gethostbyname!")); + return NULL; + } + + addrs = (struct in_addr **) my_hostent->h_addr_list; + + for(i = 0; addrs[i] != NULL; i++) { + strcpy(ip , inet_ntoa(*addrs[i]) ); + break; + } + } #ifdef WIN32 @@ -117,7 +134,6 @@ #endif - // TODO: fetch ip address for the hostname if (hpc_host != NULL) { - hostaddr = inet_addr(hpc_host->value); + hostaddr = inet_addr(ip); } else { setMapInMaps(conf,"lenv","message","No host parameter found in your main.cfg file!\n"); @@ -235,5 +251,5 @@ int ssh_get_cnt(maps* conf){ int result=0; - map* myMap=getMapFromMaps(conf,"lenv","cnt_session"); + map* myMap=getMapFromMaps(conf,"lenv","nb_sessions"); if(myMap!=NULL){ result=atoi(myMap->value); @@ -323,5 +339,5 @@ return false; } - + do { sessions[cnt]->sftp_session = libssh2_sftp_init(sessions[cnt]->session); @@ -391,5 +407,5 @@ * @param targetPath const char* defining the path for accessing the file on the * remote host - * @return true in case of success, false if failure occured + * @return 0 in case of success, -1 if failure occured */ int ssh_fetch(maps* conf,const char* localPath,const char* targetPath,int cnt){ @@ -414,65 +430,61 @@ fprintf(stderr, "Unable to init SFTP session\n"); - return false; + return -1; } } while (!sessions[cnt]->sftp_session); + do { + sftp_handle = libssh2_sftp_open(sessions[cnt]->sftp_session, targetPath, + LIBSSH2_FXF_READ, 0); + if (!sftp_handle) { + if (libssh2_session_last_errno(sessions[cnt]->session) != LIBSSH2_ERROR_EAGAIN) { + fprintf(stderr, " ** Unable to open file with SFTP\n"); + return -1; + } + else { + //non-blocking open + waitsocket(sessions[cnt]->sock_id, sessions[cnt]->session); + } + } + } while (!sftp_handle); + + int result=0; + do { do { - sftp_handle = libssh2_sftp_open(sessions[cnt]->sftp_session, targetPath, - - LIBSSH2_FXF_READ, 0); - - if (!sftp_handle) { - if (libssh2_session_last_errno(sessions[cnt]->session) != LIBSSH2_ERROR_EAGAIN) { - fprintf(stderr, "Unable to open file with SFTP\n"); - return -1; - } - else { - fprintf(stderr, "non-blocking open\n"); - waitsocket(sessions[cnt]->sock_id, sessions[cnt]->session); - } - } - } while (!sftp_handle); - - int result=1; - do { - do { - rc = libssh2_sftp_read(sftp_handle, mem, sizeof(mem)); - /*fprintf(stderr, "libssh2_sftp_read returned %d\n", - rc);*/ - if(rc > 0) { - //write(2, mem, rc); - fwrite(mem, rc, 1, local); - } - } while (rc > 0); - - if(rc != LIBSSH2_ERROR_EAGAIN) { - result=-1; - break; - } - - struct timeval timeout; - fd_set fd; - timeout.tv_sec = 10; - timeout.tv_usec = 0; - - FD_ZERO(&fd); - - FD_SET(sessions[cnt]->sock_id, &fd); - - rc = select(sessions[cnt]->sock_id+1, &fd, &fd, NULL, &timeout); - if(rc <= 0) { - if(rc==0) - fprintf(stderr, "SFTP download timed out: %d\n", rc); - else - fprintf(stderr, "SFTP download error: %d\n", rc); - result=-1; - break; - } - - } while (1); + rc = libssh2_sftp_read(sftp_handle, mem, sizeof(mem)); + /*fprintf(stderr, "libssh2_sftp_read returned %d\n", + rc);*/ + if(rc > 0) { + //write(2, mem, rc); + fwrite(mem, rc, 1, local); + } + } while (rc > 0); + + if(rc != LIBSSH2_ERROR_EAGAIN) { + result=-1; + break; + } + + struct timeval timeout; + fd_set fd; + timeout.tv_sec = 10; + timeout.tv_usec = 0; + + FD_ZERO(&fd); + + FD_SET(sessions[cnt]->sock_id, &fd); + + rc = select(sessions[cnt]->sock_id+1, &fd, &fd, NULL, &timeout); + if(rc <= 0) { + if(rc==0) + fprintf(stderr, "SFTP download timed out: %d\n", rc); + else + fprintf(stderr, "SFTP download error: %d\n", rc); + return -1; + } + + } while (1); duration = (int)(time(NULL)-start); fclose(local); libssh2_sftp_close_handle(sftp_handle); - libssh2_sftp_shutdown(sessions[cnt]->sftp_session); return 0; @@ -491,19 +503,33 @@ int exitcode; char *exitsignal=(char *)"none"; + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); while( (channel = libssh2_channel_open_session(sessions[cnt]->session)) == NULL && libssh2_session_last_error(sessions[cnt]->session,NULL,NULL,0) == LIBSSH2_ERROR_EAGAIN ) { + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); waitsocket(sessions[cnt]->sock_id, sessions[cnt]->session); } + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); if( channel == NULL ){ fprintf(stderr,"Error\n"); - return 1; - } + return -1; + } + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); while( (rc = libssh2_channel_exec(channel, command)) == LIBSSH2_ERROR_EAGAIN ) { + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); waitsocket(sessions[cnt]->sock_id, sessions[cnt]->session); } + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); if( rc != 0 ) { fprintf(stderr,"Error\n"); return -1; } + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); map* tmpPath=getMapFromMaps(conf,"main","tmpPath"); @@ -513,4 +539,6 @@ FILE* logFile=fopen(logPath,"wb"); free(logPath); + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); while(true){ int rc; @@ -535,8 +563,16 @@ break; } + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); fclose(logFile); + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); exitcode = 127; + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); while( (rc = libssh2_channel_close(channel)) == LIBSSH2_ERROR_EAGAIN ) waitsocket(sessions[cnt]->sock_id, sessions[cnt]->session); + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); if( rc == 0 ) { @@ -545,4 +581,6 @@ NULL, NULL, NULL, NULL, NULL); } + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); if (exitsignal) @@ -550,4 +588,6 @@ else fprintf(stderr, "\nEXIT: %d bytecount: %d\n", exitcode, bytecount); + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); libssh2_channel_free(channel); @@ -643,8 +683,8 @@ } #ifdef DEBUG - fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fprintf(stderr,"*** %s %d\n",__FILE__,__LINE__); fflush(stderr); dumpMaps(getMaps(*conf,"uploadQueue")); - fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fprintf(stderr,"*** %s %d\n",__FILE__,__LINE__); fflush(stderr); #endif @@ -660,6 +700,21 @@ getMapArray(queueMaps->content,"targetPath",i) }; - fprintf(stderr,"%s %d %s %s\n",__FILE__,__LINE__,argv[1]->value,argv[2]->value); - ssh_copy(*conf,argv[1]->value,argv[2]->value,ssh_get_cnt(*conf)); + fprintf(stderr,"*** %s %d %s %s\n",__FILE__,__LINE__,argv[1]->value,argv[2]->value); + /**/zooLock* lck; + if((lck=lockFile(*conf,argv[1]->value,'w'))!=NULL){/**/ + if(ssh_copy(*conf,argv[1]->value,argv[2]->value,ssh_get_cnt(*conf))!=true){ + char* templateStr=_("Unable to copy over SSH the file requested for setting the value of %s."); + char *tmpMessage=(char*)malloc((strlen(templateStr)+strlen(argv[0]->value)+1)*sizeof(char)); + sprintf(tmpMessage,templateStr,argv[0]->value); + setMapInMaps(*conf,"lenv","message",tmpMessage); + free(tmpMessage); + unlockFile(*conf,lck); + return false; + } + /**/unlockFile(*conf,lck); + }else{ + setMapInMaps(*conf,"lenv","message",_("Unable to lock the file for upload!")); + return false; + }/**/ } } Index: branches/prototype-v0/zoo-project/zoo-kernel/sshapi.h =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/sshapi.h (revision 851) +++ branches/prototype-v0/zoo-project/zoo-kernel/sshapi.h (revision 854) @@ -57,5 +57,6 @@ #include #endif - + +#include #include #include Index: branches/prototype-v0/zoo-project/zoo-kernel/ulinet.c =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/ulinet.c (revision 851) +++ branches/prototype-v0/zoo-project/zoo-kernel/ulinet.c (revision 854) @@ -211,6 +211,53 @@ ret.agent=strdup(lpszAgent); ret.nb=0; + ret.waitingRequests[ret.nb] = NULL; ret.ihandle[ret.nb].header=NULL; + ret.ihandle[ret.nb].handle=NULL; + ret.ihandle[ret.nb].hasCacheFile=0; + ret.ihandle[ret.nb].nDataAlloc = 0; + ret.ihandle[ret.nb].url = NULL; + ret.ihandle[ret.nb].mimeType = NULL; + ret.ihandle[ret.nb].cookie = NULL; + ret.ihandle[ret.nb].nDataLen = 0; + ret.ihandle[ret.nb].nDataAlloc = 0; + ret.ihandle[ret.nb].pabyData = NULL; + ret.ihandle[ret.nb].post = NULL; return ret; +} + +/** + * Verify if the URL should use a shared cache or not. + * + * In case the security section contains a key named "shared", then if the + * domain listed in the shared key are contained in the url given as parameter + * then it return "SHARED" in other cases, it returns "OTHER". + * + * @param conf the main configuration file maps + * @param url the URL to evaluate + * @return a string "SHARED" in case the host is in a domain listed in the + * shared key, "OTHER" in other cases. + */ +char* getProvenance(maps* conf,const char* url){ + map* sharedCache=getMapFromMaps(conf,"security","shared"); + if(sharedCache!=NULL){ + char *res=NULL; + char *hosts=sharedCache->value; + char *curs=strtok(hosts,","); + while(curs!=NULL){ + fprintf(stderr,"### %s %d %s \n",__FILE__,__LINE__,strstr(url,curs)); + fflush(stderr); + if(strstr(url,curs)==NULL) + res="OTHER"; + else{ + res="SHARED"; + fprintf(stderr,"### %s %d %s \n",__FILE__,__LINE__,"SHARED"); + fflush(stderr); + return res; + } + curs=strtok(NULL,","); + } + return res; + } + return "OTHER"; } @@ -282,5 +329,5 @@ int i; for(i=0;inb;i++){ - if(targetHosts->value[0]=='*' || isProtectedHost(targetHosts->value,handle->ihandle[i].url)==1){ + if(strstr(targetHosts->value,"*")!=NULL || isProtectedHost(targetHosts->value,handle->ihandle[i].url)==1){ while (token != NULL){ int length=strlen(token)+6; @@ -297,8 +344,10 @@ } fprintf(stderr,"%s %d %s \n",__FILE__,__LINE__,tmp1); - if(strncmp(tmp1,"HTTP_COOKIE",11)!=0){ - tmpMap = getMapFromMaps(conf,"renv",tmp1); - if(tmpMap!=NULL) - AddMissingHeaderEntry(&handle->ihandle[i],token,tmpMap->value); + //dumpMaps(conf); + tmpMap = getMapFromMaps(conf,"renv",tmp1); + if(tmpMap!=NULL){ + fprintf(stderr,"add header %s %s\n",token,tmpMap->value); + fflush(stderr); + AddMissingHeaderEntry(&handle->ihandle[i],token,tmpMap->value); } free(tmp1); @@ -321,42 +370,46 @@ void InternetCloseHandle(HINTERNET* handle0){ int i=0; - for(i=0;inb;i++){ - _HINTERNET handle=handle0->ihandle[i]; - if(handle.hasCacheFile>0){ - fclose(handle.file); - unlink(handle.filename); - } - else{ - handle.pabyData = NULL; - handle.nDataAlloc = handle.nDataLen = 0; - } - if(handle0->ihandle[i].header!=NULL){ - curl_slist_free_all(handle0->ihandle[i].header); - handle0->ihandle[i].header=NULL; - } - if(handle.post!=NULL) - free(handle.post); - if(handle.url!=NULL) - free(handle.url); - if(handle.mimeType!=NULL) - free(handle.mimeType); - if(handle.cookie!=NULL) - free(handle.cookie); - } - fprintf(stderr,"%s%d\n",__FILE__,__LINE__); - fflush(stderr); - if(handle0->handle) - curl_multi_cleanup(handle0->handle); - fprintf(stderr,"%s%d\n",__FILE__,__LINE__); - fflush(stderr); - free(handle0->agent); - fprintf(stderr,"%s%d\n",__FILE__,__LINE__); - fflush(stderr); - for(i=handle0->nb-1;i>=0;i--){ - fprintf(stderr,"%s%d\n",__FILE__,__LINE__); - fflush(stderr); - free(handle0->waitingRequests[i]); - fprintf(stderr,"%s%d\n",__FILE__,__LINE__); - fflush(stderr); + if(handle0!=NULL){ + for(i=0;inb;i++){ + _HINTERNET handle=handle0->ihandle[i]; + if(handle.hasCacheFile>0){ + fclose(handle.file); + unlink(handle.filename); + } + else{ + handle.pabyData = NULL; + handle.nDataAlloc = handle.nDataLen = 0; + } + if(handle.header!=NULL){ + curl_slist_free_all(handle.header); + handle.header=NULL; + } + if(handle.post!=NULL){ + free(handle.post); + handle.post=NULL; + } + if(handle.url!=NULL){ + free(handle.url); + handle.url=NULL; + } + if(handle.mimeType!=NULL){ + free(handle.mimeType); + handle.mimeType=NULL; + } + if(handle.cookie!=NULL){ + free(handle.cookie); + handle.cookie=NULL; + } + if(handle0->waitingRequests[i]!=NULL){ + free(handle0->waitingRequests[i]); + handle0->waitingRequests[i]=NULL; + } + } + if(handle0->handle) + curl_multi_cleanup(handle0->handle); + if(handle0->agent!=NULL){ + free(handle0->agent); + handle0->agent=NULL; + } } } @@ -377,24 +430,56 @@ struct MemoryStruct header; + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); hInternet->ihandle[hInternet->nb].handle=curl_easy_init( ); + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); hInternet->ihandle[hInternet->nb].hasCacheFile=0; + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); hInternet->ihandle[hInternet->nb].nDataAlloc = 0; + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); hInternet->ihandle[hInternet->nb].url = NULL; + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); hInternet->ihandle[hInternet->nb].mimeType = NULL; + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); hInternet->ihandle[hInternet->nb].cookie = NULL; + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); hInternet->ihandle[hInternet->nb].nDataLen = 0; + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); hInternet->ihandle[hInternet->nb].id = hInternet->nb; + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); hInternet->ihandle[hInternet->nb].nDataAlloc = 0; + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); hInternet->ihandle[hInternet->nb].pabyData = NULL; + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); hInternet->ihandle[hInternet->nb].post = NULL; + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_COOKIEFILE, "ALL"); + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); #ifndef TIGER curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_COOKIELIST, "ALL"); #endif curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_USERAGENT, hInternet->agent); + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle,CURLOPT_FOLLOWLOCATION,1); + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle,CURLOPT_MAXREDIRS,3); + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); header.memory=NULL; @@ -402,5 +487,9 @@ curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_HEADERFUNCTION, header_write_data); + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_WRITEHEADER, (void *)&header); + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); #ifdef MSG_LAF_VERBOSE @@ -431,4 +520,6 @@ break; } + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); #ifdef ULINET_DEBUG fprintf(stderr,"URL (%s)\nBODY (%s)\n",lpszUrl,lpszHeaders); @@ -448,14 +539,25 @@ curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle,CURLOPT_POSTFIELDSIZE,(long)dwHeadersLength); } + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); if(hInternet->ihandle[hInternet->nb].header!=NULL) curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle,CURLOPT_HTTPHEADER,hInternet->ihandle[hInternet->nb].header); + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle,CURLOPT_URL,lpszUrl); hInternet->ihandle[hInternet->nb].url = zStrdup(lpszUrl); + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); curl_multi_add_handle(hInternet->handle,hInternet->ihandle[hInternet->nb].handle); + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); hInternet->ihandle[hInternet->nb].header=NULL; + fprintf(stderr,"%s %d\n",__FILE__,__LINE__); + fflush(stderr); ++hInternet->nb; + hInternet->ihandle[hInternet->nb].header=NULL; #ifdef ULINET_DEBUG @@ -481,10 +583,16 @@ for(i=0;inb;i++){ char *tmp; + fprintf(stderr," *** %s %d %d \n",__FILE__,__LINE__,i); curl_easy_getinfo(hInternet->ihandle[i].handle,CURLINFO_CONTENT_TYPE,&tmp); + fprintf(stderr," *** %s %d %d \n",__FILE__,__LINE__,i); if(tmp!=NULL) hInternet->ihandle[i].mimeType=strdup(tmp); + fprintf(stderr," *** %s %d %d \n",__FILE__,__LINE__,i); curl_easy_getinfo(hInternet->ihandle[i].handle,CURLINFO_RESPONSE_CODE,&hInternet->ihandle[i].code); + fprintf(stderr," *** %s %d %d \n",__FILE__,__LINE__,i); curl_multi_remove_handle(hInternet->handle, hInternet->ihandle[i].handle); + fprintf(stderr," *** %s %d %d \n",__FILE__,__LINE__,i); curl_easy_cleanup(hInternet->ihandle[i].handle); + fprintf(stderr," *** %s %d %d \n",__FILE__,__LINE__,i); //fprintf(stderr,"%s %d %s \n",__FILE__,__LINE__,hInternet->ihandle[i].mimeType); //fprintf(stderr,"%s %d %s \n",__FILE__,__LINE__,hInternet->ihandle[i].pabyData); Index: branches/prototype-v0/zoo-project/zoo-kernel/ulinet.h =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/ulinet.h (revision 851) +++ branches/prototype-v0/zoo-project/zoo-kernel/ulinet.h (revision 854) @@ -128,4 +128,5 @@ HINTERNET InternetOpen(char*,int,char*,char*,int); + char* getProvenance(maps*,const char*); int isProtectedHost(const char*,const char*); int AddMissingHeaderEntry(_HINTERNET*,const char*,const char*); Index: branches/prototype-v0/zoo-project/zoo-kernel/zoo_loader.c =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/zoo_loader.c (revision 851) +++ branches/prototype-v0/zoo-project/zoo-kernel/zoo_loader.c (revision 854) @@ -86,4 +86,10 @@ fflush(stderr); #endif + fprintf (stderr, "Addr:%s\n", cgiRemoteAddr); + fprintf (stderr, "RequestMethod: (%s) %d %d\n", cgiRequestMethod,strncasecmp(cgiRequestMethod,"post",4),strncmp(cgiContentType,"text/xml",8)==0 || strncasecmp(cgiRequestMethod,"post",4)==0); + fprintf (stderr, "Request: %s\n", cgiQueryString); + fprintf (stderr, "ContentType: %s\n", cgiContentType); + fprintf (stderr, "ContentLength: %d\n", cgiContentLength); + fflush(stderr); char *strQuery=NULL; Index: branches/prototype-v0/zoo-project/zoo-kernel/zoo_service_loader.c =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/zoo_service_loader.c (revision 851) +++ branches/prototype-v0/zoo-project/zoo-kernel/zoo_service_loader.c (revision 854) @@ -107,4 +107,5 @@ #include #include +#include #include #ifndef WIN32 @@ -122,4 +123,14 @@ #include #include + +#include +#include +#include +#include + +#include +#include +#include +#include #ifndef WIN32 @@ -385,4 +396,5 @@ sig_handler (int sig) { + char tmp[100]; const char *ssig; @@ -413,5 +425,5 @@ sprintf (tmp, _ - ("ZOO Kernel failed to process your request, receiving signal %d = %s"), + ("ZOO Kernel failed to process your request, receiving signal %d = %s "), sig, ssig); errorException (NULL, tmp, "InternalError", NULL); @@ -990,7 +1002,7 @@ #ifdef ETC_DIR #ifndef WIN32 - getcwd (ntmp, 1024); + getcwd (ntmp, 1024); #else - _getcwd (ntmp, 1024); + _getcwd (ntmp, 1024); #endif #endif @@ -1184,5 +1196,5 @@ fprintf(stderr," ** DEBUG %s %d \n",__FILE__,__LINE__); fflush(stderr); - dumpMap(r_inputs); + //dumpMap(r_inputs); fprintf(stderr," ** DEBUG %s %d \n",__FILE__,__LINE__); fflush(stderr); @@ -1294,4 +1306,6 @@ } + dumpMap(request_inputs); + if (strncasecmp (REQUEST, "GetCapabilities", 15) == 0) { @@ -1348,5 +1362,4 @@ #ifdef META_DB fetchServicesFromDb(zooRegistry,m,doc,n,printGetCapabilitiesForProcess,1); - fprintf(stderr,"************************* %s %d\n\n",__FILE__,__LINE__); close_sql(m,0); #endif @@ -1430,4 +1443,6 @@ r_inputs = NULL; r_inputs = getMap (request_inputs, "version"); + fprintf(stderr," ** DEBUG %s %d \n",__FILE__,__LINE__); + fflush(stderr); xmlNodePtr n = printWPSHeader(doc,m,"DescribeProcess", root_nodes[vid][1],(version!=NULL?version->value:"1.0.0"),1); @@ -1519,5 +1534,5 @@ free (REQUEST); closedir (dirp); - xmlFreeDoc (doc); + //xmlFreeDoc (doc); xmlCleanupParser (); zooXmlCleanupNs (); @@ -1550,8 +1565,11 @@ addToMap (request_inputs, "metapath", tmpMap->value); map *tmpMapI = getMapFromMaps (m, "lenv", "Identifier"); + /** + * No support for dot in service name stored in metadb!? #ifdef META_DB - service* s2=extractServiceFromDb(m,tmpMapI->name,0); + service* s2=extractServiceFromDb(m,tmpMapI->value,0); if(s2==NULL){ #endif + */ s1 = (service *) malloc (SERVICE_SIZE); t = readServiceFile (m, buff1, &s1, tmpMapI->value); @@ -1585,5 +1603,5 @@ free (s1); closedir (dirp); - xmlFreeDoc (doc); + //xmlFreeDoc (doc); xmlCleanupParser (); zooXmlCleanupNs (); @@ -1604,7 +1622,9 @@ hasVal = 1; setMapInMaps (m, "lenv", "level", "0"); + /* #ifdef META_DB } #endif + */ } else @@ -1688,5 +1708,5 @@ free (REQUEST); closedir (dirp); - xmlFreeDoc (doc); + //xmlFreeDoc (doc); xmlCleanupParser (); zooXmlCleanupNs (); @@ -1736,5 +1756,5 @@ free (REQUEST); closedir (dirp); - xmlFreeDoc (doc); + //xmlFreeDoc (doc); xmlCleanupParser (); zooXmlCleanupNs (); @@ -1843,7 +1863,5 @@ } s1 = NULL; - if(postRequest!=NULL) - setMapInMaps (m, "lenv", "xrequest", postRequest->value); - + r_inputs = getMap (request_inputs, "Identifier"); map* import = getMapFromMaps (m, IMPORTSERVICE, r_inputs->value); @@ -1880,8 +1898,9 @@ fprintf(stderr,"CONNECTING METADB!\n"); //FAILED CONNECTING DB - if(getMapFromMaps(m,"lenv","dbIssue")!=NULL){ + if(getMapFromMaps(m,"lenv","dbIssue")!=NULL || metadb_id<0){ fprintf(stderr,"ERROR CONNECTING METADB\n"); } - s1=extractServiceFromDb(m,r_inputs->value,0); + if(metadb_id>=0) + s1=extractServiceFromDb(m,r_inputs->value,0); //close_sql(m,0); if(s1!=NULL){ @@ -2181,7 +2200,9 @@ { int hasValidCookie = -1; - char *tcook = zStrdup (cgiCookie); + char *tcook = originalCookie = zStrdup (cgiCookie); + //fprintf(stderr,">>>>> %s %d %s\n",__FILE__,__LINE__,tcook ); char *tmp = NULL; map *testing = getMapFromMaps (m, "main", "cookiePrefix"); + parseCookie(&m,originalCookie); if (testing == NULL) { @@ -2197,5 +2218,5 @@ { char *token, *saveptr; - token = strtok_r (cgiCookie, ";", &saveptr); + token = strtok_r (tcook, ";", &saveptr); while (token != NULL) { @@ -2244,7 +2265,12 @@ if (istat == 0 && file_status.st_size > 0) { + int saved_stdout = dup (fileno (stdout)); + dup2 (fileno (stderr), fileno (stdout)); conf_read (session_file_path, tmpSess); addMapsToMaps (&m, tmpSess); freeMaps (&tmpSess); + fflush(stdout); + dup2 (saved_stdout, fileno (stdout)); + close(saved_stdout); } free (tmpSess); @@ -2273,6 +2299,6 @@ char* tmpName=zStrdup(s); char* tmpValue=strstr(s,"=")+1; - char* tmpName1=(char*)malloc((1+(len-strlen(tmpValue)))*sizeof(char)); - snprintf(tmpName1,(len-strlen(tmpValue))+1,"%s",tmpName); + char* tmpName1=(char*)malloc((1+(len-(strlen(tmpValue)+1)))*sizeof(char)); + snprintf(tmpName1,(len-strlen(tmpValue)),"%s",tmpName); if(_tmpMaps->content == NULL) _tmpMaps->content = createMap (tmpName1,tmpValue); @@ -2285,15 +2311,12 @@ } if(_tmpMaps->content!=NULL && getMap(_tmpMaps->content,"HTTP_COOKIE")!=NULL){ - /*map* tmpMap1=getMap(_tmpMaps->content,"HTTP_COOKIE"); - free(tmpMap1->value); - tmpMap1->value=zStrdup(cgiCookie);*/ - fprintf(stderr,"[%s]\n",cgiCookie); addToMap(_tmpMaps->content,"HTTP_COOKIE1",&cgiCookie[0]); - dumpMap(_tmpMaps->content); } addMapsToMaps (&m, _tmpMaps); freeMaps (&_tmpMaps); free (_tmpMaps); - + if(postRequest!=NULL) + setMapInMaps (m, "renv", "xrequest", postRequest->value); + //dumpMaps(m); #ifdef WIN32 char *cgiSidL = NULL; @@ -2334,5 +2357,4 @@ "ZooWPSClient\0", INTERNET_OPEN_TYPE_PRECONFIG, NULL, NULL, 0);*/ - if(validateRequest(&m,s1,request_inputs, &request_input_real_format,&request_output_real_format,&hInternet)<0){ freeService (&s1); @@ -2350,16 +2372,15 @@ return -1; } - fprintf(stderr,"*************************\n\n"); - //fprintf(stderr,"%s \n",json_object_to_json_string_ext(mapsToJson(request_input_real_format),JSON_C_TO_STRING_PLAIN)); - fprintf(stderr,"*************************\n\n"); loadServiceAndRun (&m, s1, request_inputs, &request_input_real_format, &request_output_real_format, &eres); -#ifdef RELY_ON_DB + #ifdef META_DB - close_sql(m,1); + close_sql(m,0); +#endif + + /*#ifdef RELY_ON_DB + //close_sql(m,1); //end_sql(); -#endif - close_sql(m,0); -#endif + #endif*/ } else @@ -2387,4 +2408,5 @@ } #endif + //InternetCloseHandle (&hInternet); if (pid > 0) { @@ -2411,4 +2433,15 @@ setMapInMaps (m, "lenv", "async","true"); map* r_inputs1 = createMap("ServiceName", s1->name); + + /*hInternet = InternetOpen ( +#ifndef WIN32 + (LPCTSTR) +#endif + "ZooWPSClient\0", + INTERNET_OPEN_TYPE_PRECONFIG, NULL, NULL, 0); +#ifndef WIN32 + if (!CHECK_INET_HANDLE (hInternet)) + fprintf (stderr, "WARNING : hInternet handle failed to initialize"); + #endif*/ // Create the filename for the result file (.res) @@ -2525,13 +2558,34 @@ invokeCallback(m,request_input_real_format,NULL,1,0); fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + dumpMaps(request_output_real_format); if(validateRequest(&m,s1,request_inputs, &request_input_real_format,&request_output_real_format,&hInternet)<0){ freeService (&s1); free (s1); + fflush (stdout); + fflush (stderr); fclose (f0); fclose (f1); if(dumpBackFinalFile(m,fbkp,fbkp1)<0) return -1; + //dumpBackFinalFile(m,fbkp,fbkp1); +#ifndef RELY_ON_DB + dumpMapsToFile(bmap,fbkpres,1); + removeShmLock (m, 1); +#else + recordResponse(m,fbkp1); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + invokeCallback(m,NULL,NULL,7,0); + fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); +#endif unlink (fbkpid); unhandleStatus (m); +#ifdef RELY_ON_DB +#ifdef META_DB + cleanupCallbackThreads(); + close_sql(m,1); + //end_sql(); +#endif + close_sql(m,0); +#endif freeMaps (&m); free (m); @@ -2544,15 +2598,22 @@ freeMaps (&tmpmaps); free (tmpmaps); - fflush (stdout); - fflush (stderr); return -1; } + dumpMaps(request_output_real_format); fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); invokeCallback(m,request_input_real_format,NULL,1,1); fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); - setMapInMaps(m,"lenv","ds_nb","1"); - loadServiceAndRun (&m, s1, request_inputs, - &request_input_real_format, - &request_output_real_format, &eres); + if(getMapFromMaps(m,"lenv","mapError")!=NULL){ + setMapInMaps(m,"lenv","message",_("Issue with geographic data")); + invokeCallback(m,NULL,NULL,7,0); + eres=-1;//SERVICE_FAILED; + }else{ + setMapInMaps(m,"lenv","ds_nb","0"); + close_sql(m,0); + loadServiceAndRun (&m, s1, request_inputs, + &request_input_real_format, + &request_output_real_format, &eres); + setMapInMaps(m,"lenv","ds_nb","2"); + } } else @@ -2573,4 +2634,6 @@ fprintf (stderr, "RUN IN BACKGROUND MODE %s %d \n",__FILE__,__LINE__); #endif +fflush(stdout); +rewind(stdout); if (eres != -1) outputResponse (s1, request_input_real_format, @@ -2595,9 +2658,9 @@ if (((int) getpid ()) != cpid || cgiSid != NULL) { - fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + if (eres == SERVICE_SUCCEEDED) + invokeCallback(m,NULL,request_output_real_format,5,1); + fflush(stderr); - invokeCallback(m,NULL,request_output_real_format,5,1); - fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); - fflush(stderr); + fflush(stdout); fclose (stdout); @@ -2619,5 +2682,5 @@ setMapInMaps(m,"lenv","fstate",wpsStatus[0]); break; - } + } #ifndef RELY_ON_DB dumpMapsToFile(bmap,fbkpres,1); @@ -2625,7 +2688,6 @@ #else recordResponse(m,fbkp1); - fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); - invokeCallback(m,NULL,request_output_real_format,6,0); - fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); + if (eres == SERVICE_SUCCEEDED) + invokeCallback(m,NULL,request_output_real_format,6,0); #endif freeMaps(&bmap); @@ -2637,12 +2699,10 @@ cleanupCallbackThreads(); close_sql(m,1); - //end_sql(); #endif close_sql(m,0); -#endif - unlink (flog); + end_sql(); +#endif free(fbkpid); free(fbkpres); - free (flog); free (fbkp1); // free (tmps1); // tmps1 is stack memory and should not be freed @@ -2653,4 +2713,6 @@ fflush(stderr); fclose (stderr); + unlink (flog); + free (flog); } else{ Index: branches/prototype-v0/zoo-project/zoo-services/gdal/dem/cgi-env/Gdal_Dem.zcfg =================================================================== --- branches/prototype-v0/zoo-project/zoo-services/gdal/dem/cgi-env/Gdal_Dem.zcfg (revision 851) +++ branches/prototype-v0/zoo-project/zoo-services/gdal/dem/cgi-env/Gdal_Dem.zcfg (revision 854) @@ -44,4 +44,13 @@ + [co] + Title = Creation Option + Abstract = string representing options (NAME=VALUE) + minOccurs = 0 + maxOccurs = 1 + + DataType = string + + Index: branches/prototype-v0/zoo-project/zoo-services/gdal/dem/service.c =================================================================== --- branches/prototype-v0/zoo-project/zoo-services/gdal/dem/service.c (revision 851) +++ branches/prototype-v0/zoo-project/zoo-services/gdal/dem/service.c (revision 854) @@ -2486,4 +2486,19 @@ setMapInMaps(outputs,"Result","value",tmpMap->value); } + + tmpMap=getMapFromMaps(inputs,"co","value"); + if(tmpMap!=NULL){ + papszCreateOptions = CSLAddString( papszCreateOptions, tmpMap->value ); + map* tmpMap1; + maps* tmpMaps=getMaps(inputs,"co"); + if((tmpMap1=getMapFromMaps(inputs,"co","length"))!=NULL){ + int i=1; + int length=atoi(tmpMap1->value); + for(;icontent,"value",i); + papszCreateOptions = CSLAddString( papszCreateOptions, tmpMap->value ); + } + } + } tmpMap=NULL; Index: branches/prototype-v0/zoo-project/zoo-services/utils/hpc/Makefile =================================================================== --- branches/prototype-v0/zoo-project/zoo-services/utils/hpc/Makefile (revision 851) +++ branches/prototype-v0/zoo-project/zoo-services/utils/hpc/Makefile (revision 854) @@ -4,5 +4,5 @@ cgi-env/wps_hpc.zo: service.c - g++ ${CFLAGS} -shared -fpic -o cgi-env/wps_hpc.zo ./service.c ${GDAL_LIBS} ${XML2LDFLAGS} ${MACOS_LD_FLAGS} ${ZOO_LDFLAGS} ${MACOS_LD_NET_FLAGS} ${XSLT_LDFLAGS} ${FCGI_LDFLAGS} -L${ZRPATH}/zoo-kernel/ -lzoo_service ${ZRPATH}/zoo-kernel/sshapi.o ${HPCLDFLAGS} + g++ ${CFLAGS} -shared -fpic -o cgi-env/wps_hpc.zo service.c service2.c ${GDAL_LIBS} ${XML2LDFLAGS} ${MACOS_LD_FLAGS} ${ZOO_LDFLAGS} ${MACOS_LD_NET_FLAGS} ${XSLT_LDFLAGS} ${FCGI_LDFLAGS} -L${ZRPATH}/zoo-kernel/ -lzoo_service ${ZRPATH}/zoo-kernel/sshapi.o ${HPCLDFLAGS} install: Index: branches/prototype-v0/zoo-project/zoo-services/utils/hpc/service.c =================================================================== --- branches/prototype-v0/zoo-project/zoo-services/utils/hpc/service.c (revision 851) +++ branches/prototype-v0/zoo-project/zoo-services/utils/hpc/service.c (revision 854) @@ -3,4 +3,8 @@ * * Copyright 2008-2009 GeoLabs SARL. All rights reserved. + * + * This work was supported by public funds received in the framework of GEOSUD, + * a project (ANR-10-EQPX-20) of the program "Investissements d'Avenir" managed + * by the French National Research Agency * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -27,4 +31,5 @@ #include "service_internal.h" #include "sshapi.h" +#include "server_internal.h" #include @@ -57,10 +62,33 @@ int fd,rc=NULL; int i=0; + map* usid=getMapFromMaps(conf,"lenv","usid"); + map* tmpPath=getMapFromMaps(conf,"main","tmpPath"); + + char *flenv = + (char *) malloc ((strlen (tmpPath->value) + + strlen (jobid->value) + 12) * sizeof (char)); + sprintf (flenv, "%s/%s_lenv.cfg", tmpPath->value, jobid->value); + maps* m = (maps *) malloc (MAPS_SIZE); + m->child=NULL; + m->next=NULL; + map* configId=NULL; + + + if(conf_read(flenv, m) != 2){ + configId=getMapFromMaps(m,"lenv","configId"); + setMapInMaps(conf,"lenv","configId",configId->value); + }else{ + setMapInMaps(conf,"lenv","message",_("Unable to read the lenv section file of the requested jobid")); + return SERVICE_FAILED; + } + unlink(flenv); + free(flenv); + SSHCON *test=ssh_connect(conf); if(test==NULL){ - return SERVICE_FAILED; - } - map* usid=getMapFromMaps(conf,"lenv","usid"); - map* tmpPath=getMapFromMaps(conf,"main","tmpPath"); + setMapInMaps(conf,"lenv","message",_("Unable to connect using through ssh.")); + return SERVICE_FAILED; + } + char *logPath=(char*)malloc((strlen(tmpPath->value)+strlen(jobid->value)+12)*sizeof(char)); sprintf(logPath,"%s/exec_out_%s",tmpPath->value,jobid->value); @@ -80,5 +108,5 @@ } free(logPath); - // Run scontrol to see if the service execution ends + // Run scontrol to check if the service execution ended. // Store all the informations returned by scontrol command as a cfg file to // be parsed back by the ZOO-Kernel waiting for the execution of the remote @@ -86,9 +114,9 @@ maps* tmpMaps=createMaps("henv"); char* command=(char*)malloc((126)*sizeof(char)); - //memset(&command,0,34); - sprintf(command,"scontrol show jobid | grep -A24 %s",fcontent); + sprintf(command,"scontrol show jobid | grep -A24 JobId=%s",fcontent); if(ssh_exec(conf,command,ssh_get_cnt(conf))==0){ free(command); setMapInMaps(conf,"lenv","message",_("Failed to run scontrol remotely")); + // TODO: check status in db and if available continue in other case return SERVICE_FAILED return SERVICE_FAILED; }else{ @@ -98,67 +126,53 @@ int ts=stat(logPath, &f_status); if(ts==0) { - fcontent=(char*)malloc(sizeof(char)*(f_status.st_size+1)); - FILE* f=fopen(logPath,"rb"); - fread(fcontent,f_status.st_size,1,f); - int fsize=f_status.st_size; - fcontent[fsize]=0; - fclose(f); - free(logPath); - fprintf(stderr,"%s \n",fcontent); - char *token, *saveptr; - token = strtok_r (fcontent, " ", &saveptr); - while (token != NULL) - { - //fprintf(stderr,"%s %d\n",token,__LINE__); - char *token1, *saveptr1; - char *tmpToken=strdup(token); - token1 = strtok_r (tmpToken, "=", &saveptr1); - int isNext=-1; - int hasTwoElements=0; - char *name=NULL; - while (token1 != NULL) - { - if(hasTwoElements==0) - name=strdup(token1); - if(hasTwoElements<1) - hasTwoElements+=1; - else{ - char *value=strdup(token1); - if(value[strlen(value)-1]=='\n') - value[strlen(value)-1]=0; - if(strlen(name)>0 && strlen(value)>0){ - if(tmpMaps->content==NULL) - tmpMaps->content=createMap(name,value); - else - addToMap(tmpMaps->content,name,value); - free(value); - } - free(name); - hasTwoElements=0; - } - token1 = strtok_r (NULL, "=", &saveptr1); - } - free(tmpToken); - token = strtok_r (NULL, " ", &saveptr); - } + fcontent=(char*)malloc(sizeof(char)*(f_status.st_size+1)); + FILE* f=fopen(logPath,"rb"); + fread(fcontent,f_status.st_size,1,f); + int fsize=f_status.st_size; + fcontent[fsize]=0; + fclose(f); + free(logPath); + char *token, *saveptr; + token = strtok_r (fcontent, " ", &saveptr); + while (token != NULL) + { + char *token1, *saveptr1; + char *tmpToken=strdup(token); + token1 = strtok_r (tmpToken, "=", &saveptr1); + int isNext=-1; + int hasTwoElements=0; + char *name=NULL; + while (token1 != NULL) + { + if(hasTwoElements==0) + name=strdup(token1); + if(hasTwoElements<1) + hasTwoElements+=1; + else{ + char *value=strdup(token1); + if(value[strlen(value)-1]=='\n') + value[strlen(value)-1]=0; + if(strlen(name)>0 && strlen(value)>0){ + if(tmpMaps->content==NULL) + tmpMaps->content=createMap(name,value); + else + addToMap(tmpMaps->content,name,value); + free(value); + } + free(name); + hasTwoElements=0; + } + token1 = strtok_r (NULL, "=", &saveptr1); + } + free(tmpToken); + token = strtok_r (NULL, " ", &saveptr); + } }else{ - setMapInMaps(conf,"lenv","message",_("Unable to access the downloaded execution log file")); - return SERVICE_FAILED; + setMapInMaps(conf,"lenv","message",_("Unable to access the downloaded execution log file")); + return SERVICE_FAILED; } } - logPath=(char*)malloc((strlen(tmpPath->value)+strlen(usid->value)+15)*sizeof(char)); - sprintf(logPath,"%s/exec_status_%s",tmpPath->value,usid->value); - dumpMapsToFile(tmpMaps,logPath,0); - - char *sname=(char*)malloc((strlen(tmpPath->value)+strlen(jobid->value)+21)); - sprintf(sname,"%s/.wait_socket_%s.sock",tmpPath->value,jobid->value); - if ( (fd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) { - perror("socket error"); - setMapInMaps(conf,"lenv","message",_("Socket error")); - return SERVICE_FAILED; - } - } - logPath=(char*)malloc((strlen(tmpPath->value)+strlen(usid->value)+15)*sizeof(char)); - sprintf(logPath,"%s/exec_status_%s",tmpPath->value,usid->value); + logPath=(char*)malloc((strlen(tmpPath->value)+strlen(jobid->value)+15)*sizeof(char)); + sprintf(logPath,"%s/exec_status_%s",tmpPath->value,jobid->value); dumpMapsToFile(tmpMaps,logPath,0); char *sname=(char*)malloc((strlen(tmpPath->value)+strlen(jobid->value)+21)); @@ -179,8 +193,8 @@ if (write(fd, "3", 1) != rc) { if (rc < 0) { - perror("write error"); - setMapInMaps(conf,"lenv","message",_("Unable to announce the successful execution of the HPC service")); - close(fd); - return SERVICE_FAILED; + perror("write error"); + setMapInMaps(conf,"lenv","message",_("Unable to announce the successful execution of the HPC service")); + close(fd); + return SERVICE_FAILED; } } Index: branches/prototype-v0/zoo-project/zoo-services/utils/hpc/service2.c =================================================================== --- branches/prototype-v0/zoo-project/zoo-services/utils/hpc/service2.c (revision 854) +++ branches/prototype-v0/zoo-project/zoo-services/utils/hpc/service2.c (revision 854) @@ -0,0 +1,203 @@ +/** + * Author : Gérald FENOY + * + * Copyright 2008-2009 GeoLabs SARL. All rights reserved. + * + * This work was supported by public funds received in the framework of GEOSUD, + * a project (ANR-10-EQPX-20) of the program "Investissements d'Avenir" managed + * by the French National Research Agency + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + + +#include "service.h" +#include "service_internal.h" + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +extern "C" { + + /** + * Try to delete a cache file. + * + * @param storage the full path to store the file + * @param filename the filename to store + * @param ext the extention of the file + * @return 0 in case of success + */ + int tryDeleteCacheFile(const char* storage,const char* filename,const char* ext){ + char *filename_noext=(char*)malloc((strlen(filename)-3)*sizeof(char)); + snprintf(filename_noext,strlen(filename)-4,"%s",filename); + char* filename_full=(char*)malloc((strlen(filename)+1)*sizeof(char)); + sprintf(filename_full,"%s.%s",filename_noext,ext); + char* fullpath=(char*)malloc((strlen(storage)+strlen(filename_full)+2)*sizeof(char)); + sprintf(fullpath,"%s/%s",storage,filename_full); + if(unlink(fullpath)==0){ + // TODO store the filename_full in the deletedfiles + fprintf(stderr,"#### DeleteData #### %s %d %s has been successfully deleted\n",__FILE__,__LINE__,filename_full); + }else{ + fprintf(stderr,"#### DeleteData #### unable to delete %s \n",fullpath); + } + free(fullpath); + free(filename_full); + return 0; + } + + /** + * Try to delete a data file. + * + * @param storage the full path to store the file + * @param filename the filename to store + * @return 0 in case of success + */ + int tryDeleteDataFile(const char* storage,const char* filename){ + char* fullpath=(char*)malloc((strlen(storage)+strlen(filename)+2)*sizeof(char)); + sprintf(fullpath,"%s/%s",storage,filename); + if(unlink(fullpath)==0){ + // TODO store the filename_full in the deletedfiles + fprintf(stderr,"#### DeleteData #### %s %d %s has been successfully deleted\n",__FILE__,__LINE__,filename); + }else{ + fprintf(stderr,"#### DeleteData #### unable to delete %s \n",fullpath); + } + free(fullpath); + return 0; + } + + + /** + * DeleteData ZOO-Service : + * This service is used in the ZOO-Project to delete the data file + * associated with input or output. + */ + ZOO_DLL_EXPORT int DeleteData(maps*& conf,maps*& inputs,maps*& outputs){ + map* dataPath=getMapFromMaps(conf,"main","dataPath"); + map* tmpPath=getMapFromMaps(conf,"main","tmpPath"); + map* cacheDir=getMapFromMaps(conf,"main","cacheDir"); + map* jobid=getMapFromMaps(inputs,"jobid","value"); + map* filename=getMapFromMaps(inputs,"filename","value"); + map* ioname=getMapFromMaps(inputs,"ioname","value"); + char tmp0[4]; + sprintf(tmp0,"%c%c%c",filename->value[strlen(filename->value)-4],filename->value[strlen(filename->value)-3],filename->value[strlen(filename->value)-2]); + char *cfilename=NULL; + if(strcasecmp(tmp0,"zca")==0){ + cfilename=(char*) malloc((strlen(filename->value)+strlen(cacheDir->value)+2)*sizeof(char)); + sprintf(cfilename,"%s/%s",cacheDir->value,filename->value); + } + else{ + cfilename=(char*) malloc((strlen(filename->value)+strlen(tmpPath->value)+2)*sizeof(char)); + sprintf(cfilename,"%s/%s",tmpPath->value,filename->value); + } + zooLock* lck=lockFile(conf,cfilename,'w'); + char** deletedfiles; + if(lck!=NULL){ + if(strcasecmp(tmp0,"zca")==0){ + // Read the zcp file to verify if it comes from a shared source + char *filename_noext=(char*)malloc((strlen(filename->value)-3)*sizeof(char)); + snprintf(filename_noext,strlen(filename->value)-4,"%s",filename->value); + char* filename_full=(char*)malloc((strlen(filename->value)+1)*sizeof(char)); + sprintf(filename_full,"%s.zcp",filename_noext); + char* fullpath=(char*)malloc((strlen(cacheDir->value)+strlen(filename_full)+2)*sizeof(char)); + sprintf(fullpath,"%s/%s",cacheDir->value,filename_full); + FILE* f0=fopen(fullpath,"rb"); + char *fcontent=NULL; + if(f0!=NULL){ + long flen; + fseek (f0, 0, SEEK_END); + flen = ftell (f0); + fseek (f0, 0, SEEK_SET); + fcontent = (char *) malloc ((flen + 1) * sizeof (char)); + fread(fcontent,flen,1,f0); + fcontent[flen]=0; + fclose(f0); + } + if(fcontent!=NULL && strcasecmp(fcontent,"SHARED")!=0){ + // Delete associated zcm and zcp + tryDeleteCacheFile(cacheDir->value,filename->value,"zca"); + tryDeleteCacheFile(cacheDir->value,filename->value,"zcm"); + tryDeleteCacheFile(cacheDir->value,filename->value,"zcp"); + // Delete ZOO_DATA__.data and _.map + char* datafile=(char*)malloc((strlen(jobid->value)+strlen(ioname->value)+19)*sizeof(char)); + sprintf(datafile,"ZOO_DATA_%s_%s.data",ioname->value,jobid->value); + tryDeleteDataFile(dataPath->value,datafile); + free(datafile); + datafile=(char*)malloc((strlen(jobid->value)+strlen(ioname->value)+19)*sizeof(char)); + sprintf(datafile,"%s_%s.map",ioname->value,jobid->value); + tryDeleteDataFile(dataPath->value,datafile); + free(datafile); + }else{ + setMapInMaps(conf,"lenv","message",_ss("The file you try to delete is a shared ressource and cannot be deleted by nature.")); + unlockFile(conf,lck); + return SERVICE_FAILED; + } + setMapInMaps(outputs,"Result","value",_ss("The input data has been correclty removed")); + }else{ + char tmp1[8]; + snprintf(tmp1,7,"%s",filename->value); + if(strcasecmp(tmp1,"output")==0){ + tryDeleteDataFile(tmpPath->value,filename->value); + // Delete ZOO_DATA__.data and _.map + char* datafile=(char*)malloc((strlen(jobid->value)+strlen(ioname->value)+19)*sizeof(char)); + sprintf(datafile,"ZOO_DATA_%s_%s.data",ioname->value,jobid->value); + tryDeleteDataFile(dataPath->value,datafile); + free(datafile); + datafile=(char*)malloc((strlen(jobid->value)+strlen(ioname->value)+19)*sizeof(char)); + sprintf(datafile,"%s_%s.map",ioname->value,jobid->value); + tryDeleteDataFile(dataPath->value,datafile); + free(datafile); + char* webServices[3]={"wms","wfs","wcs"}; + int i=0; + // Delete ZOO_DATA__link_.data and _link_.map + // with is the corresponding OGC web service (wms,wfs,wcs) + for(i=0;i<3;i++){ + datafile=(char*)malloc((strlen(jobid->value)+24)*sizeof(char)); + sprintf(datafile,"ZOO_DATA_%s_link_%s.data",webServices[i],jobid->value); + tryDeleteDataFile(dataPath->value,datafile); + free(datafile); + datafile=(char*)malloc((strlen(jobid->value)+14)*sizeof(char)); + sprintf(datafile,"%s_link_%s.map",webServices[i],jobid->value); + tryDeleteDataFile(dataPath->value,datafile); + free(datafile); + } + setMapInMaps(outputs,"Result","value",_ss("The output data has been correclty removed")); + }else{ + setMapInMaps(conf,"lenv","message",_ss("The file you try to delete is nor an input, nor and output.")); + unlockFile(conf,lck); + return SERVICE_FAILED; + } + } + unlockFile(conf,lck); + } + else{ + setMapInMaps(conf,"lenv","message",_ss("Failed to acquire lock for deletion, please try again later.")); + return SERVICE_FAILED; + } + return SERVICE_SUCCEEDED; + } + +} Index: branches/prototype-v0/zoo-project/zoo-services/utils/status/Makefile =================================================================== --- branches/prototype-v0/zoo-project/zoo-services/utils/status/Makefile (revision 851) +++ branches/prototype-v0/zoo-project/zoo-services/utils/status/Makefile (revision 854) @@ -4,5 +4,5 @@ cgi-env/wps_status.zo: service.c - g++ ${CFLAGS} -shared -fpic -o cgi-env/wps_status.zo ./service.c ${GDAL_LIBS} ${XML2LDFLAGS} ${MACOS_LD_FLAGS} ${ZOO_LDFLAGS} ${MACOS_LD_NET_FLAGS} ${XSLT_LDFLAGS} ${FCGI_LDFLAGS} -L${ZRPATH}/zoo-kernel/ -lzoo_service + g++ -rdynamic ${CFLAGS} -shared -fpic -o cgi-env/wps_status.zo ./service.c ${GDAL_LIBS} ${XML2LDFLAGS} ${MACOS_LD_FLAGS} ${ZOO_LDFLAGS} ${MACOS_LD_NET_FLAGS} ${XSLT_LDFLAGS} ${FCGI_LDFLAGS} -L${ZRPATH}/zoo-kernel/ -lzoo_service install: