[822] | 1 | /* |
---|
| 2 | * Author : Gérald FENOY |
---|
| 3 | * |
---|
[839] | 4 | * Copyright (c) 2017 GeoLabs SARL |
---|
[822] | 5 | * |
---|
[839] | 6 | * This work was supported by public funds received in the framework of GEOSUD, |
---|
| 7 | * a project (ANR-10-EQPX-20) of the program "Investissements d'Avenir" managed |
---|
| 8 | * by the French National Research Agency |
---|
| 9 | * |
---|
[822] | 10 | * Permission is hereby granted, free of charge, to any person obtaining a copy |
---|
| 11 | * of this software and associated documentation files (the "Software"), to deal |
---|
| 12 | * in the Software without restriction, including without limitation the rights |
---|
| 13 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
---|
| 14 | * copies of the Software, and to permit persons to whom the Software is |
---|
| 15 | * furnished to do so, subject to the following conditions: |
---|
| 16 | * |
---|
| 17 | * The above copyright notice and this permission notice shall be included in |
---|
| 18 | * all copies or substantial portions of the Software. |
---|
| 19 | * |
---|
| 20 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
---|
| 21 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
---|
| 22 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
---|
| 23 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
---|
| 24 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
---|
| 25 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
---|
| 26 | * THE SOFTWARE. |
---|
| 27 | * |
---|
| 28 | */ |
---|
| 29 | |
---|
| 30 | #include "service_internal_hpc.h" |
---|
| 31 | #include "response_print.h" |
---|
| 32 | #include "server_internal.h" |
---|
[839] | 33 | #include "service_callback.h" |
---|
| 34 | #include "mimetypes.h" |
---|
[822] | 35 | #include <sys/un.h> |
---|
| 36 | |
---|
[839] | 37 | typedef struct { |
---|
| 38 | maps* conf; |
---|
| 39 | char* local_file; |
---|
| 40 | char* target_file; |
---|
| 41 | } local_params; |
---|
| 42 | |
---|
[846] | 43 | /** |
---|
| 44 | * Add nested outputs to every outputs that is geographic format |
---|
| 45 | * @see isGeographic |
---|
| 46 | * @param s the service current definition |
---|
| 47 | */ |
---|
[839] | 48 | void addNestedOutputs(service** s){ |
---|
| 49 | if((*s)==NULL){ |
---|
| 50 | return; |
---|
| 51 | } |
---|
| 52 | if(*s==NULL || (*s)->outputs==NULL){ |
---|
| 53 | return; |
---|
[822] | 54 | } |
---|
[839] | 55 | elements *out=(*s)->outputs; |
---|
| 56 | elements* cur=out; |
---|
| 57 | map* serviceType=getMap((*s)->content,"ServiceType"); |
---|
| 58 | if(strncmp(serviceType->value,"HPC",3)!=0) |
---|
| 59 | return; |
---|
| 60 | while(cur!=NULL && cur->defaults!=NULL){ |
---|
| 61 | map* mimeType=getMap(cur->defaults->content,"mimeType"); |
---|
| 62 | if(mimeType!=NULL){ |
---|
| 63 | int geo=isGeographic(mimeType->value); |
---|
| 64 | if(geo>0){ |
---|
| 65 | elements *tmp[3]={ |
---|
| 66 | dupElements(cur), |
---|
| 67 | dupElements(cur), |
---|
| 68 | dupElements(cur) |
---|
| 69 | }; |
---|
| 70 | char *geoLink="wcs_link"; |
---|
| 71 | if(geo==2){ |
---|
| 72 | geoLink="wfs_link"; |
---|
| 73 | } |
---|
| 74 | int i=0; |
---|
| 75 | for(;i<3;i++){ |
---|
| 76 | if(tmp[i]->next!=NULL){ |
---|
| 77 | freeElements(&tmp[i]->next); |
---|
| 78 | free(tmp[i]->next); |
---|
| 79 | tmp[i]->next=NULL; |
---|
| 80 | } |
---|
| 81 | free(tmp[i]->name); |
---|
| 82 | tmp[i]->format=zStrdup("ComplexData"); |
---|
| 83 | freeMap(&tmp[i]->content); |
---|
| 84 | free(tmp[i]->content); |
---|
| 85 | tmp[i]->content=NULL; |
---|
| 86 | switch(i){ |
---|
| 87 | case 0: |
---|
| 88 | tmp[i]->name=zStrdup("download_link"); |
---|
| 89 | tmp[i]->content=createMap("Title",_("Download link")); |
---|
| 90 | addToMap(tmp[i]->content,"Abstract",_("The download link")); |
---|
| 91 | addToMap(tmp[i]->defaults->content,"useMapserver","false"); |
---|
| 92 | if(tmp[i]->supported!=NULL){ |
---|
| 93 | freeIOType(&tmp[i]->supported); |
---|
| 94 | free(tmp[i]->supported); |
---|
| 95 | tmp[i]->supported=NULL; |
---|
| 96 | } |
---|
| 97 | break; |
---|
| 98 | case 1: |
---|
| 99 | tmp[i]->name=zStrdup("wms_link"); |
---|
| 100 | tmp[i]->content=createMap("Title",_("WMS link")); |
---|
| 101 | addToMap(tmp[i]->content,"Abstract",_("The WMS link")); |
---|
| 102 | if(tmp[i]->supported!=NULL && tmp[i]->supported->next!=NULL){ |
---|
| 103 | freeIOType(&tmp[i]->supported->next); |
---|
| 104 | free(tmp[i]->supported->next); |
---|
| 105 | tmp[i]->supported->next=NULL; |
---|
| 106 | }else{ |
---|
| 107 | if(tmp[i]->supported!=NULL) |
---|
| 108 | addToMap(tmp[i]->supported->content,"useMapserver","true"); |
---|
| 109 | addToMap(tmp[i]->defaults->content,"useMapserver","true"); |
---|
| 110 | } |
---|
| 111 | break; |
---|
| 112 | case 2: |
---|
| 113 | if(geo==2){ |
---|
| 114 | tmp[i]->name=zStrdup("wfs_link"); |
---|
| 115 | tmp[i]->content=createMap("Title",_("WFS link")); |
---|
| 116 | addToMap(tmp[i]->content,"Abstract",_("The WFS link")); |
---|
| 117 | }else{ |
---|
| 118 | tmp[i]->name=zStrdup("wcs_link"); |
---|
| 119 | tmp[i]->content=createMap("Title",_("WCS link")); |
---|
| 120 | addToMap(tmp[i]->content,"Abstract",_("The WCS link")); |
---|
| 121 | } |
---|
| 122 | if(tmp[i]->supported!=NULL && tmp[i]->supported->next!=NULL && |
---|
| 123 | tmp[i]->supported->next->content!=NULL){ |
---|
| 124 | freeIOType(&tmp[i]->supported); |
---|
| 125 | free(tmp[i]->supported); |
---|
| 126 | tmp[i]->supported=NULL; |
---|
| 127 | tmp[i]->supported=createIoType(); |
---|
| 128 | iotype* cnext=cur->supported->next; |
---|
| 129 | tmp[i]->supported->content=createMap(cnext->content->name,cnext->content->value); |
---|
| 130 | addMapToMap(&tmp[i]->supported->content,cnext->content->next); |
---|
| 131 | addToMap(tmp[i]->supported->content,"useMapserver","true"); |
---|
| 132 | }else |
---|
| 133 | addToMap(tmp[i]->defaults->content,"useMapserver","true"); |
---|
| 134 | break; |
---|
| 135 | } |
---|
| 136 | } |
---|
| 137 | |
---|
| 138 | addToElements(&cur->child,tmp[0]); |
---|
| 139 | addToElements(&cur->child,tmp[1]); |
---|
| 140 | addToElements(&cur->child,tmp[2]); |
---|
| 141 | free(cur->format); |
---|
| 142 | cur->format=NULL; |
---|
| 143 | if(cur->defaults!=NULL){ |
---|
| 144 | freeIOType(&cur->defaults); |
---|
| 145 | cur->defaults=NULL; |
---|
| 146 | } |
---|
| 147 | if(cur->supported!=NULL){ |
---|
| 148 | freeIOType(&cur->supported); |
---|
| 149 | cur->supported=NULL; |
---|
| 150 | } |
---|
| 151 | freeElements(&tmp[2]); |
---|
| 152 | free(tmp[2]); |
---|
| 153 | freeElements(&tmp[1]); |
---|
| 154 | free(tmp[1]); |
---|
| 155 | freeElements(&tmp[0]); |
---|
| 156 | free(tmp[0]); |
---|
| 157 | //addToMap(cur->content,"internal","true"); |
---|
| 158 | } |
---|
| 159 | } |
---|
| 160 | cur=cur->next; |
---|
| 161 | } |
---|
[822] | 162 | } |
---|
| 163 | |
---|
| 164 | /** |
---|
| 165 | * Load and run a HPC Application corresponding to the service. |
---|
| 166 | * |
---|
| 167 | * @param main_conf the conf maps containing the main.cfg settings |
---|
| 168 | * @param request the map containing the HTTP request |
---|
| 169 | * @param s the service structure |
---|
| 170 | * @param real_inputs the maps containing the inputs |
---|
| 171 | * @param real_outputs the maps containing the outputs |
---|
| 172 | */ |
---|
| 173 | int zoo_hpc_support(maps** main_conf,map* request,service* s,maps **real_inputs,maps **real_outputs){ |
---|
| 174 | maps* m=*main_conf; |
---|
| 175 | maps* inputs=*real_inputs; |
---|
| 176 | maps* outputs=*real_outputs; |
---|
| 177 | map* tmp0=getMapFromMaps(*main_conf,"lenv","cwd"); |
---|
| 178 | char *ntmp=tmp0->value; |
---|
| 179 | map* tmp=NULL; |
---|
| 180 | int res=-1; |
---|
[839] | 181 | char *serviceType; |
---|
| 182 | map* mServiceType=getMap(s->content,"serviceType"); |
---|
| 183 | if(mServiceType!=NULL) |
---|
| 184 | serviceType=mServiceType->value; |
---|
| 185 | else |
---|
| 186 | serviceType="HPC"; |
---|
| 187 | map* targetPathMap=getMapFromMaps(*main_conf,serviceType,"storagePath"); |
---|
[822] | 188 | map* tmpPath=getMapFromMaps(*main_conf,"main","tmpPath"); |
---|
| 189 | map* uuid=getMapFromMaps(*main_conf,"lenv","usid"); |
---|
[839] | 190 | pthread_t threads_pool[50]; |
---|
| 191 | // Force the HPC services to be called asynchronously |
---|
| 192 | map* isAsync=getMapFromMaps(*main_conf,"lenv","async"); |
---|
| 193 | if(isAsync==NULL){ |
---|
| 194 | errorException(*main_conf,_("The synchronous mode is not supported by this type of service"),"NoSuchMode",s->name); |
---|
| 195 | return -1; |
---|
| 196 | } |
---|
[822] | 197 | |
---|
| 198 | maps* input=*real_inputs; |
---|
| 199 | char **parameters=NULL; |
---|
| 200 | int parameters_cnt=0; |
---|
[839] | 201 | while(input!=NULL && input->content!=NULL){ |
---|
| 202 | if(getMaps(*real_outputs,input->name)==NULL){ |
---|
| 203 | parameters_cnt+=1; |
---|
| 204 | if(parameters_cnt==1) |
---|
| 205 | parameters=(char**)malloc(parameters_cnt*sizeof(char*)); |
---|
| 206 | else |
---|
| 207 | parameters=(char**)realloc(parameters,parameters_cnt*sizeof(char*)); |
---|
| 208 | if(getMap(input->content,"mimeType")!=NULL){ |
---|
| 209 | // Input is ComplexData |
---|
| 210 | if(getMap(input->content,"cache_file")==NULL){ |
---|
| 211 | // Input data has been passed by value |
---|
| 212 | // TODO: publish input through MapServer / use output publication |
---|
| 213 | dumpMapsValuesToFiles(main_conf,&input); |
---|
| 214 | addToMap(input->content,"toPublish","true"); |
---|
| 215 | } |
---|
| 216 | if(getMap(input->content,"cache_file")!=NULL){ |
---|
| 217 | map* length=getMap(input->content,"length"); |
---|
| 218 | if(length==NULL){ |
---|
| 219 | addToMap(input->content,"length","1"); |
---|
| 220 | length=getMap(input->content,"length"); |
---|
| 221 | } |
---|
| 222 | int len=atoi(length->value); |
---|
| 223 | int i=0; |
---|
| 224 | for(i=0;i<len;i++){ |
---|
| 225 | map* tmp=getMapArray(input->content,"cache_file",i); |
---|
| 226 | char* targetName=strrchr(tmp->value,'/'); |
---|
| 227 | char *targetPath=(char*)malloc((strlen(targetPathMap->value)+strlen(targetName)+2)*sizeof(char)); |
---|
| 228 | sprintf(targetPath,"%s/%s",targetPathMap->value,targetName); |
---|
| 229 | setMapArray(input->content,"targetPath",i,targetPath); |
---|
| 230 | setMapArray(input->content,"localPath",i,tmp->value); |
---|
| 231 | addToUploadQueue(main_conf,input); |
---|
| 232 | if(i==0){ |
---|
| 233 | parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(targetPath)+3)*sizeof(char)); |
---|
| 234 | sprintf(parameters[parameters_cnt-1],"-%s %s",input->name,targetPath); |
---|
| 235 | }else{ |
---|
| 236 | char *tmpStr=zStrdup(parameters[parameters_cnt-1]); |
---|
| 237 | parameters[parameters_cnt-1]=(char*)realloc(parameters[parameters_cnt-1],(strlen(tmpStr)+strlen(targetPath)+2)*sizeof(char)); |
---|
| 238 | sprintf(parameters[parameters_cnt-1],"%s %s",tmpStr,targetPath); |
---|
| 239 | free(tmpStr); |
---|
| 240 | } |
---|
| 241 | free(targetPath); |
---|
| 242 | } |
---|
| 243 | }else{ |
---|
| 244 | // ??? |
---|
| 245 | fprintf(stderr,"%s %d\n",__FILE__,__LINE__); |
---|
| 246 | fflush(stderr); |
---|
| 247 | } |
---|
[822] | 248 | }else{ |
---|
[839] | 249 | // LitteralData and BboxData |
---|
| 250 | if(getMap(input->content,"dataType")!=NULL){ |
---|
| 251 | // For LitteralData, simply pass the value |
---|
| 252 | map* val=getMap(input->content,"value"); |
---|
| 253 | parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(val->value)+3)*sizeof(char)); |
---|
| 254 | sprintf(parameters[parameters_cnt-1],"-%s %s",input->name,val->value); |
---|
| 255 | } |
---|
[822] | 256 | } |
---|
| 257 | } |
---|
| 258 | input=input->next; |
---|
| 259 | } |
---|
| 260 | |
---|
[839] | 261 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
| 262 | invokeCallback(m,inputs,NULL,2,0); |
---|
| 263 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
| 264 | |
---|
| 265 | // Upload data on HPC |
---|
| 266 | runUpload(main_conf); |
---|
| 267 | |
---|
| 268 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
| 269 | invokeCallback(m,inputs,NULL,2,1); |
---|
| 270 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
| 271 | |
---|
[822] | 272 | // Add the filename to generate for every output to the parameters |
---|
| 273 | input=*real_outputs; |
---|
| 274 | // TODO: fix appendOutputParameters |
---|
| 275 | //appendOutputParameters(input,parameters,¶meters_cnt,s,uuid,targetPathMap); |
---|
| 276 | while(input!=NULL){ |
---|
| 277 | // TODO: parse all outputs including inner outputs if required. |
---|
| 278 | if(input->child==NULL){ |
---|
| 279 | parameters_cnt+=1; |
---|
| 280 | if(parameters_cnt==1) |
---|
| 281 | parameters=(char**)malloc(parameters_cnt*sizeof(char*)); |
---|
| 282 | else |
---|
| 283 | parameters=(char**)realloc(parameters,parameters_cnt*sizeof(char*)); |
---|
| 284 | // Name every files that should be produced by the service execution |
---|
[839] | 285 | map* mime=getMap(input->content,"mimeType"); |
---|
| 286 | char* targetName; |
---|
| 287 | if(mime!=NULL){ |
---|
| 288 | bool hasExt=false; |
---|
| 289 | map* fileExt=getFileExtensionMap(mime->value,&hasExt); |
---|
| 290 | targetName=(char*)malloc((strlen(s->name)+strlen(input->name)+strlen(uuid->value)+strlen(fileExt->value)+11)*sizeof(char)); |
---|
| 291 | sprintf(targetName,"output_%s_%s_%s.%s",s->name,input->name,uuid->value,fileExt->value); |
---|
| 292 | freeMap(&fileExt); |
---|
| 293 | free(fileExt); |
---|
| 294 | }else{ |
---|
| 295 | targetName=(char*)malloc((strlen(s->name)+strlen(input->name)+strlen(uuid->value)+14)*sizeof(char)); |
---|
| 296 | sprintf(targetName,"output_%s_%s_%s.tif",s->name,input->name,uuid->value); |
---|
| 297 | } |
---|
[822] | 298 | char *targetPath=(char*)malloc((strlen(targetPathMap->value)+strlen(targetName)+2)*sizeof(char)); |
---|
| 299 | sprintf(targetPath,"%s/%s",targetPathMap->value,targetName); |
---|
| 300 | setMapInMaps(*real_outputs,input->name,"generated_file",targetPath); |
---|
[839] | 301 | // We should verify if any optional tag for output is required |
---|
| 302 | // (i.e. -out output.tiff *int8*), meaning that we should search |
---|
| 303 | // for a corresponding inputs name. |
---|
| 304 | map* inValue=getMapFromMaps(*real_inputs,input->name,"value"); |
---|
| 305 | if(inValue!=NULL){ |
---|
| 306 | parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(targetPath)+strlen(inValue->value)+4)*sizeof(char)); |
---|
| 307 | sprintf(parameters[parameters_cnt-1],"-%s %s %s",input->name,targetPath,inValue->value); |
---|
| 308 | }else{ |
---|
| 309 | parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(targetPath)+3)*sizeof(char)); |
---|
| 310 | sprintf(parameters[parameters_cnt-1],"-%s %s",input->name,targetPath); |
---|
| 311 | } |
---|
| 312 | free(targetPath); |
---|
| 313 | }// In other case it means we need to return the cache_file as generated_file |
---|
| 314 | else{ |
---|
| 315 | parameters_cnt+=1; |
---|
| 316 | if(parameters_cnt==1) |
---|
| 317 | parameters=(char**)malloc(parameters_cnt*sizeof(char*)); |
---|
| 318 | else |
---|
| 319 | parameters=(char**)realloc(parameters,parameters_cnt*sizeof(char*)); |
---|
| 320 | // Name every files that should be produced by the service execution |
---|
| 321 | map* mime=getMap(input->child->content,"mimeType"); |
---|
| 322 | char* targetName; |
---|
| 323 | if(mime!=NULL){ |
---|
| 324 | bool hasExt=false; |
---|
| 325 | map* fileExt=getFileExtensionMap(mime->value,&hasExt); |
---|
| 326 | targetName=(char*)malloc((strlen(s->name)+strlen(input->name)+strlen(uuid->value)+strlen(fileExt->value)+11)*sizeof(char)); |
---|
| 327 | sprintf(targetName,"output_%s_%s_%s.%s",s->name,input->name,uuid->value,fileExt->value); |
---|
| 328 | freeMap(&fileExt); |
---|
| 329 | free(fileExt); |
---|
| 330 | }else{ |
---|
| 331 | targetName=(char*)malloc((strlen(s->name)+strlen(input->name)+strlen(uuid->value)+14)*sizeof(char)); |
---|
| 332 | sprintf(targetName,"output_%s_%s_%s.tif",s->name,input->name,uuid->value); |
---|
| 333 | } |
---|
| 334 | char *targetPath=(char*)malloc((strlen(targetPathMap->value)+strlen(targetName)+2)*sizeof(char)); |
---|
| 335 | sprintf(targetPath,"%s/%s",targetPathMap->value,targetName); |
---|
| 336 | addToMap(input->content,"generated_file",targetPath); |
---|
| 337 | // We should verify if any optional tag for output is required |
---|
| 338 | // (i.e. -out output.tiff *int8*), meaning that we should search |
---|
| 339 | // for a corresponding inputs name. |
---|
| 340 | map* inValue=getMapFromMaps(*real_inputs,input->name,"value"); |
---|
| 341 | if(inValue!=NULL){ |
---|
| 342 | parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(targetPath)+strlen(inValue->value)+4)*sizeof(char)); |
---|
| 343 | sprintf(parameters[parameters_cnt-1],"-%s %s %s",input->name,targetPath,inValue->value); |
---|
| 344 | }else{ |
---|
| 345 | parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(targetPath)+3)*sizeof(char)); |
---|
| 346 | sprintf(parameters[parameters_cnt-1],"-%s %s",input->name,targetPath); |
---|
| 347 | } |
---|
| 348 | free(targetPath); |
---|
[822] | 349 | } |
---|
| 350 | input=input->next; |
---|
| 351 | } |
---|
| 352 | |
---|
[839] | 353 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
| 354 | fflush(stderr); |
---|
| 355 | invokeCallback(m,inputs,NULL,3,0); |
---|
| 356 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
| 357 | fflush(stderr); |
---|
| 358 | |
---|
[822] | 359 | // Produce the SBATCH File locally |
---|
[839] | 360 | char *scriptPath=(char*)malloc((strlen(s->name)+strlen(tmpPath->value)+strlen(uuid->value)+10)*sizeof(char)); |
---|
[822] | 361 | sprintf(scriptPath,"%s/zoo_%s_%s.sh",tmpPath->value,s->name,uuid->value); |
---|
[839] | 362 | setMapInMaps(*main_conf,"lenv","local_script",scriptPath); |
---|
| 363 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
| 364 | fflush(stderr); |
---|
| 365 | invokeCallback(m,inputs,NULL,3,0); |
---|
| 366 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
| 367 | fflush(stderr); |
---|
[822] | 368 | FILE* scriptFile=fopen(scriptPath,"w+"); |
---|
[839] | 369 | map* headerMap=getMapFromMaps(*main_conf,serviceType,"header"); |
---|
[822] | 370 | if(headerMap!=NULL){ |
---|
| 371 | // Use the header file if defined in the HPC section of the main.cfg file |
---|
| 372 | struct stat f_status; |
---|
| 373 | int s=stat(headerMap->value, &f_status); |
---|
| 374 | if(s==0){ |
---|
| 375 | char* fcontent=(char*)malloc(sizeof(char)*(f_status.st_size+1)); |
---|
| 376 | FILE* f=fopen(headerMap->value,"rb"); |
---|
| 377 | fread(fcontent,f_status.st_size,1,f); |
---|
| 378 | int fsize=f_status.st_size; |
---|
| 379 | fcontent[fsize]=0; |
---|
| 380 | fclose(f); |
---|
[839] | 381 | fprintf(scriptFile,"%s\n### --- ZOO-Service HEADER end --- ###\n\n",fcontent); |
---|
[822] | 382 | free(fcontent); |
---|
| 383 | }else |
---|
| 384 | fprintf(scriptFile,"#!/bin/bash\n\n### *** Default ZOO-Service HEADER (no header found) *** ###\n\n"); |
---|
| 385 | }else |
---|
| 386 | fprintf(scriptFile,"#!/bin/bash\n\n### *** Default ZOO-Service HEADER *** ###\n\n"); |
---|
[839] | 387 | maps* hpc_opts=getMaps(*main_conf,"sbatch_options"); |
---|
| 388 | if(hpc_opts!=NULL){ |
---|
| 389 | map* hpc_opts_content=hpc_opts->content; |
---|
| 390 | while(hpc_opts_content!=NULL){ |
---|
| 391 | fprintf(scriptFile,"#SBATCH --%s=%s\n",hpc_opts_content->name,hpc_opts_content->value); |
---|
| 392 | hpc_opts_content=hpc_opts_content->next; |
---|
| 393 | } |
---|
[822] | 394 | } |
---|
| 395 | fprintf(scriptFile,"#SBATCH --job-name=ZOO-Project_%s_%s\n\n",uuid->value,s->name); |
---|
| 396 | map* mods=getMap(s->content,"hpcModules"); |
---|
| 397 | if(mods!=NULL) |
---|
| 398 | fprintf(scriptFile,"#SBATCH --export=MODULES=%s\n",mods->value); |
---|
[839] | 399 | |
---|
| 400 | map* bodyMap=getMapFromMaps(*main_conf,serviceType,"body"); |
---|
| 401 | if(bodyMap!=NULL){ |
---|
| 402 | // Use the header file if defined in the HPC section of the main.cfg file |
---|
| 403 | struct stat f_status; |
---|
| 404 | int s=stat(bodyMap->value, &f_status); |
---|
| 405 | if(s==0){ |
---|
| 406 | char* fcontent=(char*)malloc(sizeof(char)*(f_status.st_size+1)); |
---|
| 407 | FILE* f=fopen(bodyMap->value,"rb"); |
---|
| 408 | fread(fcontent,f_status.st_size,1,f); |
---|
| 409 | int fsize=f_status.st_size; |
---|
| 410 | fcontent[fsize]=0; |
---|
| 411 | fclose(f); |
---|
| 412 | fprintf(scriptFile,"%s\n### --- ZOO-Service BODY end --- ###\n\n",fcontent); |
---|
| 413 | free(fcontent); |
---|
| 414 | }else |
---|
| 415 | fprintf(scriptFile,"#!/bin/bash\n\n### *** Default ZOO-Service BODY (no body found) *** ###\n\n"); |
---|
| 416 | }else |
---|
| 417 | fprintf(scriptFile,"#!/bin/bash\n\n### *** Default ZOO-Service BODY *** ###\n\n"); |
---|
| 418 | |
---|
| 419 | |
---|
[822] | 420 | map* sp=getMap(s->content,"serviceProvider"); |
---|
[839] | 421 | |
---|
[822] | 422 | // Require to produce the command line to be executed |
---|
| 423 | fprintf(scriptFile,"\n\necho \"Job started at: $(date)\"\n"); |
---|
| 424 | fprintf(scriptFile,"echo \"Running service: [%s]\"\n",sp->value); |
---|
| 425 | fprintf(scriptFile,"%s ",sp->value); |
---|
| 426 | for(int i=0;i<parameters_cnt;i++){ |
---|
| 427 | fprintf(scriptFile," %s",parameters[i]); |
---|
| 428 | } |
---|
[839] | 429 | for(int i=parameters_cnt-1;i>=0;i--){ |
---|
| 430 | free(parameters[i]); |
---|
| 431 | } |
---|
[822] | 432 | free(parameters); |
---|
| 433 | fprintf(scriptFile,"\n"); |
---|
| 434 | fprintf(scriptFile,"echo \"Job finished at: $(date)\"\n"); |
---|
| 435 | fflush(scriptFile); |
---|
| 436 | fclose(scriptFile); |
---|
[839] | 437 | |
---|
| 438 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
| 439 | invokeCallback(m,inputs,NULL,3,1); |
---|
| 440 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
[822] | 441 | |
---|
| 442 | // Upload the SBATCH File to the remote host |
---|
[839] | 443 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
| 444 | invokeCallback(m,inputs,NULL,4,0); |
---|
| 445 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
| 446 | targetPathMap=getMapFromMaps(*main_conf,serviceType,"executePath"); |
---|
[822] | 447 | if(targetPathMap==NULL){ |
---|
| 448 | setMapInMaps(*main_conf,"lenv","message",_("There is no executePath defined in you HPC section!")); |
---|
| 449 | return SERVICE_FAILED; |
---|
| 450 | } |
---|
| 451 | char* targetName=strrchr(scriptPath,'/'); |
---|
| 452 | char *targetPath=(char*)malloc((strlen(targetPathMap->value)+strlen(targetName)+2)*sizeof(char)); |
---|
| 453 | sprintf(targetPath,"%s/%s",targetPathMap->value,targetName); |
---|
[839] | 454 | setMapInMaps(*main_conf,"lenv","remote_script",targetPath); |
---|
[822] | 455 | SSHCON *test=ssh_connect(*main_conf); |
---|
[839] | 456 | ssh_copy(*main_conf,scriptPath,targetPath,ssh_get_cnt(*main_conf)); |
---|
[822] | 457 | |
---|
| 458 | // Execute the SBATCH script remotely |
---|
| 459 | map* subStr=getMapFromMaps(*main_conf,"HPC","subStr"); |
---|
| 460 | char *command=(char*)malloc((strlen(targetPath)+strlen(targetPathMap->value)+strlen(subStr->value)+strlen(uuid->value)+137)*sizeof(char)); |
---|
| 461 | sprintf(command,"sbatch %s 2> %s/error_%s.log | sed \"s:%s::g\"",targetPath,targetPathMap->value,uuid->value,subStr->value); |
---|
[839] | 462 | if(ssh_exec(*main_conf,command,ssh_get_cnt(m))==0){ |
---|
[822] | 463 | // The sbatch command has failed! |
---|
| 464 | // Download the error log file from the HPC server |
---|
| 465 | char tmpS[1024]; |
---|
| 466 | free(command); |
---|
[839] | 467 | command=(char*)malloc((strlen(targetPathMap->value)+strlen(uuid->value)+11)*sizeof(char)); |
---|
[822] | 468 | sprintf(command,"%s/error_%s.log",targetPathMap->value,uuid->value); |
---|
| 469 | targetName=strrchr(command,'/'); |
---|
| 470 | free(targetPath); |
---|
| 471 | targetPath=(char*)malloc((strlen(tmpPath->value)+strlen(targetName)+2)*sizeof(char)); |
---|
| 472 | sprintf(targetPath,"%s/%s",tmpPath->value,targetName); |
---|
[839] | 473 | if(ssh_fetch(*main_conf,targetPath,command,ssh_get_cnt(m))==0){ |
---|
[822] | 474 | struct stat f_status; |
---|
| 475 | int ts=stat(targetPath, &f_status); |
---|
| 476 | if(ts==0) { |
---|
| 477 | char* fcontent = NULL; |
---|
| 478 | fcontent=(char*)malloc(sizeof(char)*(f_status.st_size+1)); |
---|
| 479 | FILE* f=fopen(targetPath,"rb"); |
---|
| 480 | fread(fcontent,f_status.st_size,1,f); |
---|
| 481 | int fsize=f_status.st_size; |
---|
| 482 | fcontent[fsize]=0; |
---|
| 483 | fclose(f); |
---|
| 484 | setMapInMaps(*main_conf,"lenv","message",fcontent); |
---|
| 485 | free(fcontent); |
---|
| 486 | }else |
---|
| 487 | setMapInMaps(*main_conf,"lenv","message",_("No message provided")); |
---|
| 488 | }else |
---|
| 489 | setMapInMaps(*main_conf,"lenv","message",_("Unable to fetch the remote error log file")); |
---|
| 490 | tmpPath=getMapFromMaps(*main_conf,"lenv","message"); |
---|
[839] | 491 | invokeCallback(m,NULL,NULL,7,1); |
---|
[822] | 492 | sprintf(tmpS, "Cannot execute the HPC ZOO-Service %s: %s", s->name, tmpPath->value); |
---|
| 493 | errorException(m,tmpS,"NoApplicableCode",NULL); |
---|
| 494 | free(command); |
---|
| 495 | free(targetPath); |
---|
| 496 | ssh_close(*main_conf); |
---|
| 497 | sleep(120); |
---|
| 498 | return -1; |
---|
| 499 | } |
---|
[839] | 500 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
| 501 | fflush(stderr); |
---|
| 502 | invokeCallback(m,NULL,NULL,4,1); |
---|
| 503 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
| 504 | fflush(stderr); |
---|
| 505 | free(command); |
---|
[822] | 506 | |
---|
| 507 | struct sockaddr_un addr; |
---|
| 508 | memset(&addr, 0, sizeof(addr)); |
---|
| 509 | addr.sun_family = AF_UNIX; |
---|
| 510 | int rc, cl, fd = socket(AF_UNIX, SOCK_STREAM, 0); |
---|
[839] | 511 | char *sname=(char*)malloc((strlen(tmpPath->value)+strlen(uuid->value)+20)); |
---|
[822] | 512 | sprintf(sname,"%s/.wait_socket_%s.sock",tmpPath->value,uuid->value); |
---|
| 513 | strncpy(addr.sun_path, sname, sizeof(addr.sun_path)-1); |
---|
[839] | 514 | |
---|
[822] | 515 | if (bind(fd, (struct sockaddr*)&addr, sizeof(addr)) == -1) { |
---|
| 516 | perror("bind error"); |
---|
| 517 | sleep(120); |
---|
| 518 | return -1; |
---|
| 519 | } |
---|
| 520 | if (listen(fd, 5) == -1) { |
---|
[839] | 521 | setMapInMaps(*main_conf,"lenv","message",_("Listen error")); |
---|
[822] | 522 | return -1; |
---|
| 523 | } |
---|
[839] | 524 | if ( (cl = accept(fd, NULL, NULL)) == -1) { |
---|
| 525 | setMapInMaps(*main_conf,"lenv","message",_("Accept error")); |
---|
[822] | 526 | return -1; |
---|
[839] | 527 | }else{ |
---|
| 528 | int hasPassed=-1; |
---|
| 529 | char buf[11]; |
---|
| 530 | memset(&buf,0,11); |
---|
| 531 | while ( (rc=read(cl,buf,10)) ) { |
---|
| 532 | if(rc==0){ |
---|
| 533 | sleep(1); |
---|
| 534 | setMapInMaps(*main_conf,"lenv","message",_("Read closed")); |
---|
[822] | 535 | return -1; |
---|
[839] | 536 | }else{ |
---|
| 537 | if(rc<0){ |
---|
| 538 | setMapInMaps(*main_conf,"lenv","message",_("Read error")); |
---|
[822] | 539 | return -1; |
---|
| 540 | } |
---|
[839] | 541 | } |
---|
| 542 | hasPassed=1; |
---|
| 543 | res=atoi(buf); |
---|
| 544 | unlink(sname); |
---|
| 545 | //free(sname); |
---|
| 546 | |
---|
| 547 | if(res==3){ |
---|
| 548 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
| 549 | fflush(stderr); |
---|
| 550 | invokeCallback(m,NULL,outputs,5,0); |
---|
| 551 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
| 552 | fflush(stderr); |
---|
| 553 | input=*real_outputs; |
---|
| 554 | while(input!=NULL){ |
---|
| 555 | if(input->child==NULL){ |
---|
| 556 | map* generatedFile=getMap(input->content,"generated_file"); |
---|
| 557 | if(generatedFile!=NULL){ |
---|
| 558 | char* filename=strrchr(generatedFile->value,'/'); |
---|
| 559 | char* targetPath=(char*)malloc((strlen(tmpPath->value)+strlen(filename)+2)*sizeof(char)); |
---|
| 560 | sprintf(targetPath,"%s/%s",tmpPath->value,filename); |
---|
| 561 | test=ssh_connect(*main_conf); |
---|
| 562 | if(ssh_fetch(*main_conf,targetPath,generatedFile->value,ssh_get_cnt(m))==0){ |
---|
| 563 | setMapInMaps(*real_outputs,input->name,"generated_file",targetPath); |
---|
| 564 | free(targetPath); |
---|
| 565 | }else{ |
---|
| 566 | char *tmpStr=(char*)malloc((strlen(filename)+strlen(_("Unable to fetch the remote file for %s"))+1)*sizeof(char)); |
---|
| 567 | sprintf(tmpStr,_("Unable to fetch the remote file for %s"),filename); |
---|
| 568 | setMapInMaps(*main_conf,"lenv","message",tmpStr); |
---|
| 569 | free(tmpStr); |
---|
| 570 | return SERVICE_FAILED; |
---|
[822] | 571 | } |
---|
[839] | 572 | } |
---|
| 573 | }else{ |
---|
| 574 | fprintf(stderr,"%s %d\n",__FILE__,__LINE__); |
---|
| 575 | fflush(stderr); |
---|
| 576 | map* generatedFile=getMap(input->content,"generated_file"); |
---|
| 577 | if(generatedFile!=NULL){ |
---|
| 578 | char* filename=strrchr(generatedFile->value,'/'); |
---|
| 579 | char* targetPath=(char*)malloc((strlen(tmpPath->value)+strlen(filename)+2)*sizeof(char)); |
---|
| 580 | sprintf(targetPath,"%s/%s",tmpPath->value,filename); |
---|
| 581 | test=ssh_connect(*main_conf); |
---|
| 582 | if(ssh_fetch(*main_conf,targetPath,generatedFile->value,ssh_get_cnt(m))==0){ |
---|
| 583 | maps* tmp=getMaps(*real_outputs,input->name); |
---|
| 584 | freeMap(&tmp->content); |
---|
| 585 | free(tmp->content); |
---|
| 586 | tmp->content=NULL; |
---|
| 587 | maps* output=getMaps(*real_outputs,input->name); |
---|
| 588 | setMapInMaps(output->child,"download_link","generated_file",targetPath); |
---|
| 589 | setMapInMaps(output->child,"download_link","useMapserver","false"); |
---|
| 590 | setMapInMaps(output->child,"WMS_LINK","generated_file",targetPath); |
---|
| 591 | setMapInMaps(output->child,"WMS_LINK","useMapserver","true"); |
---|
| 592 | setMapInMaps(output->child,"WCS_LINK","generated_file",targetPath); |
---|
| 593 | setMapInMaps(output->child,"WCS_LINK","useMapserver","true"); |
---|
| 594 | } |
---|
| 595 | } |
---|
[822] | 596 | } |
---|
[839] | 597 | input=input->next; |
---|
[822] | 598 | } |
---|
| 599 | } |
---|
[839] | 600 | //free(buf); |
---|
[822] | 601 | } |
---|
[839] | 602 | if(hasPassed<0){ |
---|
| 603 | perror("Failed to read"); |
---|
| 604 | setMapInMaps(*main_conf,"lenv","message",_("Unable to parse the value returned by remote execution")); |
---|
| 605 | sleep(120); |
---|
| 606 | return SERVICE_FAILED; |
---|
| 607 | } |
---|
| 608 | } |
---|
[822] | 609 | ssh_close(*main_conf); |
---|
| 610 | return res; |
---|
| 611 | } |
---|
| 612 | |
---|
| 613 | |
---|