1 | /* |
---|
2 | * Author : Gérald FENOY |
---|
3 | * |
---|
4 | * Copyright (c) 2017 GeoLabs SARL |
---|
5 | * |
---|
6 | * This work was supported by public funds received in the framework of GEOSUD, |
---|
7 | * a project (ANR-10-EQPX-20) of the program "Investissements d'Avenir" managed |
---|
8 | * by the French National Research Agency |
---|
9 | * |
---|
10 | * Permission is hereby granted, free of charge, to any person obtaining a copy |
---|
11 | * of this software and associated documentation files (the "Software"), to deal |
---|
12 | * in the Software without restriction, including without limitation the rights |
---|
13 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
---|
14 | * copies of the Software, and to permit persons to whom the Software is |
---|
15 | * furnished to do so, subject to the following conditions: |
---|
16 | * |
---|
17 | * The above copyright notice and this permission notice shall be included in |
---|
18 | * all copies or substantial portions of the Software. |
---|
19 | * |
---|
20 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
---|
21 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
---|
22 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
---|
23 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
---|
24 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
---|
25 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
---|
26 | * THE SOFTWARE. |
---|
27 | * |
---|
28 | */ |
---|
29 | |
---|
30 | #include "service_internal_hpc.h" |
---|
31 | #include "response_print.h" |
---|
32 | #include "server_internal.h" |
---|
33 | #include "service_callback.h" |
---|
34 | #include "mimetypes.h" |
---|
35 | #include <sys/un.h> |
---|
36 | |
---|
37 | typedef struct { |
---|
38 | maps* conf; |
---|
39 | char* local_file; |
---|
40 | char* target_file; |
---|
41 | } local_params; |
---|
42 | |
---|
43 | /** |
---|
44 | * Add nested outputs to every outputs that is geographic format |
---|
45 | * @see isGeographic |
---|
46 | * @param s the service current definition |
---|
47 | */ |
---|
48 | void addNestedOutputs(service** s){ |
---|
49 | if((*s)==NULL){ |
---|
50 | return; |
---|
51 | } |
---|
52 | if(*s==NULL || (*s)->outputs==NULL || (*s)->content==NULL){ |
---|
53 | return; |
---|
54 | } |
---|
55 | elements *out=(*s)->outputs; |
---|
56 | elements* cur=out; |
---|
57 | map* serviceType=getMap((*s)->content,"ServiceType"); |
---|
58 | if(strncmp(serviceType->value,"HPC",3)!=0) |
---|
59 | return; |
---|
60 | while(cur!=NULL && cur->defaults!=NULL){ |
---|
61 | map* mimeType=getMap(cur->defaults->content,"mimeType"); |
---|
62 | map* useMS=getMap(cur->defaults->content,"useMapserver"); |
---|
63 | if(mimeType!=NULL && useMS!=NULL && strncasecmp(useMS->value,"true",4)==0){ |
---|
64 | int geo=isGeographic(mimeType->value); |
---|
65 | if(geo>0){ |
---|
66 | elements *tmp[3]={ |
---|
67 | dupElements(cur), |
---|
68 | dupElements(cur), |
---|
69 | dupElements(cur) |
---|
70 | }; |
---|
71 | char *geoLink="wcs_link"; |
---|
72 | if(geo==2){ |
---|
73 | geoLink="wfs_link"; |
---|
74 | } |
---|
75 | int i=0; |
---|
76 | for(;i<3;i++){ |
---|
77 | if(tmp[i]->next!=NULL){ |
---|
78 | freeElements(&tmp[i]->next); |
---|
79 | free(tmp[i]->next); |
---|
80 | tmp[i]->next=NULL; |
---|
81 | } |
---|
82 | free(tmp[i]->name); |
---|
83 | if(tmp[i]->format!=NULL) |
---|
84 | free(tmp[i]->format); |
---|
85 | tmp[i]->format=zStrdup("ComplexData"); |
---|
86 | freeMap(&tmp[i]->content); |
---|
87 | free(tmp[i]->content); |
---|
88 | tmp[i]->content=NULL; |
---|
89 | switch(i){ |
---|
90 | case 0: |
---|
91 | tmp[i]->name=zStrdup("download_link"); |
---|
92 | tmp[i]->content=createMap("Title",_("Download link")); |
---|
93 | addToMap(tmp[i]->content,"Abstract",_("The download link")); |
---|
94 | addToMap(tmp[i]->defaults->content,"useMapserver","false"); |
---|
95 | if(tmp[i]->supported!=NULL){ |
---|
96 | freeIOType(&tmp[i]->supported); |
---|
97 | free(tmp[i]->supported); |
---|
98 | tmp[i]->supported=NULL; |
---|
99 | } |
---|
100 | break; |
---|
101 | case 1: |
---|
102 | tmp[i]->name=zStrdup("wms_link"); |
---|
103 | tmp[i]->content=createMap("Title",_("WMS link")); |
---|
104 | addToMap(tmp[i]->content,"Abstract",_("The WMS link")); |
---|
105 | if(tmp[i]->supported!=NULL && tmp[i]->supported->next!=NULL){ |
---|
106 | freeIOType(&tmp[i]->supported->next); |
---|
107 | free(tmp[i]->supported->next); |
---|
108 | tmp[i]->supported->next=NULL; |
---|
109 | }else{ |
---|
110 | if(tmp[i]->supported!=NULL) |
---|
111 | addToMap(tmp[i]->supported->content,"useMapserver","true"); |
---|
112 | addToMap(tmp[i]->defaults->content,"useMapserver","true"); |
---|
113 | } |
---|
114 | break; |
---|
115 | case 2: |
---|
116 | if(geo==2){ |
---|
117 | tmp[i]->name=zStrdup("wfs_link"); |
---|
118 | tmp[i]->content=createMap("Title",_("WFS link")); |
---|
119 | addToMap(tmp[i]->content,"Abstract",_("The WFS link")); |
---|
120 | }else{ |
---|
121 | tmp[i]->name=zStrdup("wcs_link"); |
---|
122 | tmp[i]->content=createMap("Title",_("WCS link")); |
---|
123 | addToMap(tmp[i]->content,"Abstract",_("The WCS link")); |
---|
124 | } |
---|
125 | if(tmp[i]->supported!=NULL && tmp[i]->supported->next!=NULL && |
---|
126 | tmp[i]->supported->next->content!=NULL){ |
---|
127 | freeIOType(&tmp[i]->supported); |
---|
128 | free(tmp[i]->supported); |
---|
129 | tmp[i]->supported=NULL; |
---|
130 | tmp[i]->supported=createIoType(); |
---|
131 | iotype* cnext=cur->supported->next; |
---|
132 | tmp[i]->supported->content=createMap(cnext->content->name,cnext->content->value); |
---|
133 | addMapToMap(&tmp[i]->supported->content,cnext->content->next); |
---|
134 | addToMap(tmp[i]->supported->content,"useMapserver","true"); |
---|
135 | }else |
---|
136 | addToMap(tmp[i]->defaults->content,"useMapserver","true"); |
---|
137 | break; |
---|
138 | } |
---|
139 | } |
---|
140 | addToElements(&cur->child,tmp[0]); |
---|
141 | addToElements(&cur->child,tmp[1]); |
---|
142 | addToElements(&cur->child,tmp[2]); |
---|
143 | free(cur->format); |
---|
144 | cur->format=NULL; |
---|
145 | if(cur->defaults!=NULL){ |
---|
146 | freeIOType(&cur->defaults); |
---|
147 | free(cur->defaults); |
---|
148 | cur->defaults=NULL; |
---|
149 | } |
---|
150 | if(cur->supported!=NULL){ |
---|
151 | freeIOType(&cur->supported); |
---|
152 | free(cur->supported); |
---|
153 | cur->supported=NULL; |
---|
154 | } |
---|
155 | freeElements(&tmp[2]); |
---|
156 | free(tmp[2]); |
---|
157 | freeElements(&tmp[1]); |
---|
158 | free(tmp[1]); |
---|
159 | freeElements(&tmp[0]); |
---|
160 | free(tmp[0]); |
---|
161 | //addToMap(cur->content,"internal","true"); |
---|
162 | } |
---|
163 | }else{ |
---|
164 | if(mimeType!=NULL){ |
---|
165 | elements *tmp=dupElements(cur); |
---|
166 | tmp->name=zStrdup("download_link"); |
---|
167 | tmp->content=createMap("Title",_("Download link")); |
---|
168 | addToMap(tmp->content,"Abstract",_("The download link")); |
---|
169 | addToMap(tmp->defaults->content,"useMapserver","false"); |
---|
170 | if(tmp->supported!=NULL){ |
---|
171 | freeIOType(&tmp->supported); |
---|
172 | free(tmp->supported); |
---|
173 | tmp->supported=NULL; |
---|
174 | } |
---|
175 | addToElements(&cur->child,tmp); |
---|
176 | free(cur->format); |
---|
177 | cur->format=NULL; |
---|
178 | if(cur->defaults!=NULL){ |
---|
179 | freeIOType(&cur->defaults); |
---|
180 | free(cur->defaults); |
---|
181 | cur->defaults=NULL; |
---|
182 | } |
---|
183 | if(cur->supported!=NULL){ |
---|
184 | freeIOType(&cur->supported); |
---|
185 | free(cur->supported); |
---|
186 | cur->supported=NULL; |
---|
187 | } |
---|
188 | freeElements(&tmp); |
---|
189 | free(tmp); |
---|
190 | } |
---|
191 | } |
---|
192 | cur=cur->next; |
---|
193 | } |
---|
194 | //dumpElements((*s)->outputs); |
---|
195 | } |
---|
196 | |
---|
197 | /** |
---|
198 | * Acquire a read lock on every files used as input for executing a service. |
---|
199 | * @param conf the main configuration file map |
---|
200 | * @return 0 if every file can be locked, -1 if one lock has failed. |
---|
201 | */ |
---|
202 | int addReadLocks(maps** conf){ |
---|
203 | map* queueLengthMap=getMapFromMaps(*conf,"uploadQueue","length"); |
---|
204 | maps* queueMaps=getMaps(*conf,"uploadQueue"); |
---|
205 | if(queueLengthMap!=NULL){ |
---|
206 | int cnt=atoi(queueLengthMap->value); |
---|
207 | int i=0; |
---|
208 | for(i=0;i<cnt;i++){ |
---|
209 | map* argv[2]={ |
---|
210 | getMapArray(queueMaps->content,"input",i), |
---|
211 | getMapArray(queueMaps->content,"localPath",i) |
---|
212 | }; |
---|
213 | zooLock* lck; |
---|
214 | if((lck=lockFile(*conf,argv[1]->value,'r'))==NULL){ |
---|
215 | char* templateStr=_("Unable to lock the file for %s in read mode."); |
---|
216 | char *tmpMessage=(char*)malloc((strlen(templateStr)+strlen(argv[0]->value)+1)*sizeof(char)); |
---|
217 | sprintf(tmpMessage,templateStr,argv[0]->value); |
---|
218 | setMapInMaps(*conf,"lenv","message",tmpMessage); |
---|
219 | free(tmpMessage); |
---|
220 | return -1; |
---|
221 | }else{ |
---|
222 | if(zoo_file_locks_cnt==0){ |
---|
223 | zoo_file_locks=(zooLock**)malloc(sizeof(zooLock*)); |
---|
224 | } |
---|
225 | else{ |
---|
226 | zoo_file_locks=(zooLock**)realloc(zoo_file_locks,(zoo_file_locks_cnt+1)*sizeof(zooLock*)); |
---|
227 | } |
---|
228 | zoo_file_locks[zoo_file_locks_cnt]=lck; |
---|
229 | zoo_file_locks_cnt++; |
---|
230 | } |
---|
231 | } |
---|
232 | } |
---|
233 | return 0; |
---|
234 | } |
---|
235 | |
---|
236 | /** |
---|
237 | * Remove all read locks set for files used as input for executing the service. |
---|
238 | * @param conf the main configuration maps pointer |
---|
239 | * @return 0 in case of success, -1 if any error occured. In case of error, one |
---|
240 | * can refer to the message map array from the lenv section. |
---|
241 | */ |
---|
242 | int removeReadLocks(maps** conf){ |
---|
243 | int res=0; |
---|
244 | int nberr=0; |
---|
245 | map* queueLengthMap=getMapFromMaps(*conf,"uploadQueue","length"); |
---|
246 | maps* queueMaps=getMaps(*conf,"uploadQueue"); |
---|
247 | if(queueLengthMap!=NULL){ |
---|
248 | int cnt=atoi(queueLengthMap->value); |
---|
249 | int i=0; |
---|
250 | for(i=0;i<cnt;i++){ |
---|
251 | if(unlockFile(*conf,zoo_file_locks[i])<1){ |
---|
252 | map* argv=getMapArray(queueMaps->content,"input",i); |
---|
253 | char* templateStr=_("Unable to unlock the file for %s after execution."); |
---|
254 | char *tmpMessage=(char*)malloc((strlen(templateStr)+strlen(argv->value)+1)*sizeof(char)); |
---|
255 | sprintf(tmpMessage,templateStr,argv->value); |
---|
256 | maps* lenv=getMaps(*conf,"lenv"); |
---|
257 | setMapArray(lenv->content,"message",nberr,tmpMessage); |
---|
258 | free(tmpMessage); |
---|
259 | res=-1; |
---|
260 | nberr++; |
---|
261 | } |
---|
262 | } |
---|
263 | } |
---|
264 | free(zoo_file_locks); |
---|
265 | return res; |
---|
266 | } |
---|
267 | |
---|
268 | /** |
---|
269 | * Get the section name depending on number of features and/or pixels of each |
---|
270 | * inputs and the threshold defined in a section. |
---|
271 | * It supposes that your inputs has been published using MapServer support, |
---|
272 | * implying that the number of features (nb_features), respectively pixels |
---|
273 | * (nb_pixels), are defined. The section, identified by confId, should contain |
---|
274 | * preview_max_features and preview_max_pixels defining the threshold values. |
---|
275 | * @param conf the main configuration file maps pointer |
---|
276 | * @param inputs the inputs maps pointer |
---|
277 | * @param confId the section identifier |
---|
278 | * @return "preview_conf" in case the numbers are lower than the threshold, |
---|
279 | * "fullres_conf" in other cases. |
---|
280 | */ |
---|
281 | char* getConfiguration(maps** conf,maps** inputs,const char* confId){ |
---|
282 | maps* input=*inputs; |
---|
283 | map* max_pixels=getMapFromMaps(*conf,confId,"preview_max_pixels"); |
---|
284 | map* max_features=getMapFromMaps(*conf,confId,"preview_max_features"); |
---|
285 | int i_max_pixels=atoi(max_pixels->value); |
---|
286 | int i_max_features=atoi(max_features->value); |
---|
287 | while(input!=NULL && input->content!=NULL){ |
---|
288 | map* tmpMap=getMap(input->content,"geodatatype"); |
---|
289 | if(tmpMap!=NULL){ |
---|
290 | map* currentNb; |
---|
291 | if(strcasecmp(tmpMap->value,"raster")==0 ){ |
---|
292 | currentNb=getMap(input->content,"nb_pixels"); |
---|
293 | if(atoi(currentNb->value)>i_max_pixels) |
---|
294 | return "fullres_conf"; |
---|
295 | }else{ |
---|
296 | if(strcasecmp(tmpMap->value,"vector")==0 ){ |
---|
297 | currentNb=getMap(input->content,"nb_features"); |
---|
298 | if(atoi(currentNb->value)>i_max_features) |
---|
299 | return "fullres_conf"; |
---|
300 | } |
---|
301 | } |
---|
302 | } |
---|
303 | input=input->next; |
---|
304 | } |
---|
305 | return "preview_conf"; |
---|
306 | } |
---|
307 | |
---|
308 | /** |
---|
309 | * Load and run a HPC Application corresponding to the service. |
---|
310 | * |
---|
311 | * @param main_conf the conf maps containing the main.cfg settings |
---|
312 | * @param request the map containing the HTTP request |
---|
313 | * @param s the service structure |
---|
314 | * @param real_inputs the maps containing the inputs |
---|
315 | * @param real_outputs the maps containing the outputs |
---|
316 | * @return SERVICE_SUCCEEDED in case of success, -1 or SERVICE_FAILED when failing. |
---|
317 | */ |
---|
318 | int zoo_hpc_support(maps** main_conf,map* request,service* s,maps **real_inputs,maps **real_outputs){ |
---|
319 | maps* m=*main_conf; |
---|
320 | maps* inputs=*real_inputs; |
---|
321 | maps* outputs=*real_outputs; |
---|
322 | map* tmp0=getMapFromMaps(*main_conf,"lenv","cwd"); |
---|
323 | char *ntmp=tmp0->value; |
---|
324 | map* tmp=NULL; |
---|
325 | int res=-1; |
---|
326 | // Get the configuration id depending on service type and defined thresholds |
---|
327 | // then, set the configId key in the lenv section |
---|
328 | char *serviceType; |
---|
329 | map* mServiceType=getMap(s->content,"confId"); |
---|
330 | if(mServiceType!=NULL) |
---|
331 | serviceType=mServiceType->value; |
---|
332 | else |
---|
333 | serviceType="HPC"; |
---|
334 | map* tmpPath=getMapFromMaps(*main_conf,"main","tmpPath"); |
---|
335 | map* uuid=getMapFromMaps(*main_conf,"lenv","usid"); |
---|
336 | map* confMap=getMapFromMaps(*main_conf,serviceType,getConfiguration(main_conf,real_inputs,serviceType)); |
---|
337 | char * configurationId=confMap->value; |
---|
338 | setMapInMaps(*main_conf,"lenv","configId",configurationId); |
---|
339 | // Dump lenv maps again after having set the configId ... |
---|
340 | char *flenv = |
---|
341 | (char *) |
---|
342 | malloc ((strlen (tmpPath->value) + |
---|
343 | strlen (uuid->value) + 12) * sizeof (char)); |
---|
344 | sprintf (flenv, "%s/%s_lenv.cfg", tmpPath->value, uuid->value); |
---|
345 | maps* lenvMaps=getMaps(m,"lenv"); |
---|
346 | dumpMapsToFile(lenvMaps,flenv,0); |
---|
347 | free(flenv); |
---|
348 | |
---|
349 | map* targetPathMap=getMapFromMaps(*main_conf,configurationId,"remote_data_path"); |
---|
350 | map* targetPersistentPathMap=getMapFromMaps(*main_conf,configurationId,"remote_persistent_data_path"); |
---|
351 | |
---|
352 | pthread_t threads_pool[50]; |
---|
353 | // Force the HPC services to be called asynchronously |
---|
354 | map* isAsync=getMapFromMaps(*main_conf,"lenv","async"); |
---|
355 | if(isAsync==NULL){ |
---|
356 | errorException(*main_conf,_("The synchronous mode is not supported by this type of service"),"NoSuchMode",s->name); |
---|
357 | return -1; |
---|
358 | } |
---|
359 | |
---|
360 | maps* input=*real_inputs; |
---|
361 | char **parameters=NULL; |
---|
362 | int parameters_cnt=0; |
---|
363 | while(input!=NULL && input->content!=NULL){ |
---|
364 | map* isInRequest=getMap(input->content,"inRequest"); |
---|
365 | map* minNb=getMap(input->content,"minOccurs"); |
---|
366 | if(getMaps(*real_outputs,input->name)==NULL && |
---|
367 | ( (isInRequest!=NULL && strncasecmp(isInRequest->value,"true",4)==0) |
---|
368 | || (minNb!=NULL && atoi(minNb->value)>0) ) ){ |
---|
369 | parameters_cnt+=1; |
---|
370 | if(parameters_cnt==1) |
---|
371 | parameters=(char**)malloc(parameters_cnt*sizeof(char*)); |
---|
372 | else |
---|
373 | parameters=(char**)realloc(parameters,parameters_cnt*sizeof(char*)); |
---|
374 | if(getMap(input->content,"mimeType")!=NULL){ |
---|
375 | // Input is ComplexData |
---|
376 | if(getMap(input->content,"cache_file")==NULL){ |
---|
377 | // Input data has been passed by value |
---|
378 | // TODO: publish input through MapServer / use output publication |
---|
379 | dumpMapsValuesToFiles(main_conf,&input); |
---|
380 | addToMap(input->content,"toPublish","true"); |
---|
381 | //addToMap(input->content,"useMapserver","true"); |
---|
382 | } |
---|
383 | if(getMap(input->content,"cache_file")!=NULL){ |
---|
384 | map* length=getMap(input->content,"length"); |
---|
385 | if(length==NULL){ |
---|
386 | addToMap(input->content,"length","1"); |
---|
387 | length=getMap(input->content,"length"); |
---|
388 | } |
---|
389 | int len=atoi(length->value); |
---|
390 | int i=0; |
---|
391 | for(i=0;i<len;i++){ |
---|
392 | map* tmp=getMapArray(input->content,"cache_file",i); |
---|
393 | map* origin=getMapArray(input->content,"origin",i); |
---|
394 | char* targetName=strrchr(tmp->value,'/'); |
---|
395 | char *targetPath; |
---|
396 | if(origin!=NULL && strcasecmp(origin->value,"SHARED")==0 && targetPersistentPathMap!=NULL){ |
---|
397 | targetPath=(char*)malloc((strlen(targetPersistentPathMap->value)+strlen(targetName)+2)*sizeof(char)); |
---|
398 | sprintf(targetPath,"%s/%s",targetPersistentPathMap->value,targetName); |
---|
399 | } |
---|
400 | else{ |
---|
401 | targetPath=(char*)malloc((strlen(targetPathMap->value)+strlen(targetName)+2)*sizeof(char)); |
---|
402 | sprintf(targetPath,"%s/%s",targetPathMap->value,targetName); |
---|
403 | } |
---|
404 | setMapArray(input->content,"targetPath",i,targetPath); |
---|
405 | setMapArray(input->content,"localPath",i,tmp->value); |
---|
406 | map* tmp1=getMapArray(input->content,"value",i); |
---|
407 | if(tmp1!=NULL){ |
---|
408 | free(tmp1->value); |
---|
409 | tmp1->value=strdup("empty"); |
---|
410 | } |
---|
411 | if(i==0){ |
---|
412 | parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(targetPath)+3)*sizeof(char)); |
---|
413 | sprintf(parameters[parameters_cnt-1],"-%s %s",input->name,targetPath); |
---|
414 | }else{ |
---|
415 | fprintf(stderr,"%s %d\n",__FILE__,__LINE__); |
---|
416 | fflush(stderr); |
---|
417 | char *tmpStr=zStrdup(parameters[parameters_cnt-1]); |
---|
418 | parameters[parameters_cnt-1]=(char*)realloc(parameters[parameters_cnt-1],(strlen(tmpStr)+strlen(targetPath)+2)*sizeof(char)); |
---|
419 | sprintf(parameters[parameters_cnt-1],"%s %s",tmpStr,targetPath); |
---|
420 | free(tmpStr); |
---|
421 | fprintf(stderr,"%s %d\n",__FILE__,__LINE__); |
---|
422 | fflush(stderr); |
---|
423 | } |
---|
424 | free(targetPath); |
---|
425 | } |
---|
426 | addToUploadQueue(main_conf,input); |
---|
427 | }else{ |
---|
428 | // ??? |
---|
429 | fprintf(stderr,"%s %d\n",__FILE__,__LINE__); |
---|
430 | fflush(stderr); |
---|
431 | } |
---|
432 | }else{ |
---|
433 | // LitteralData and BboxData |
---|
434 | if(getMap(input->content,"dataType")!=NULL){ |
---|
435 | // For LitteralData, simply pass the value |
---|
436 | map* length=getMap(input->content,"length"); |
---|
437 | if(length!=NULL){ |
---|
438 | char* value=NULL; |
---|
439 | int len=atoi(length->value); |
---|
440 | int i=0; |
---|
441 | for(i=0;i<len;i++){ |
---|
442 | map* val=getMapArray(input->content,"value",i); |
---|
443 | if(val!=NULL){ |
---|
444 | if(value==NULL){ |
---|
445 | value=(char*)malloc((strlen(val->value)+3)*sizeof(char)); |
---|
446 | sprintf(value,"\"%s\"",val->value); |
---|
447 | } |
---|
448 | else{ |
---|
449 | value=(char*)realloc(value,(strlen(value)+strlen(val->value)+4)*sizeof(char)); |
---|
450 | sprintf(value,"%s \"%s\"",value,val->value); |
---|
451 | } |
---|
452 | } |
---|
453 | } |
---|
454 | if(value!=NULL){ |
---|
455 | parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(value)+3)*sizeof(char)); |
---|
456 | sprintf(parameters[parameters_cnt-1],"-%s %s",input->name,value); |
---|
457 | } |
---|
458 | }else{ |
---|
459 | map* val=getMap(input->content,"value"); |
---|
460 | parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(val->value)+5)*sizeof(char)); |
---|
461 | sprintf(parameters[parameters_cnt-1],"-%s \"%s\"",input->name,val->value); |
---|
462 | } |
---|
463 | } |
---|
464 | } |
---|
465 | } |
---|
466 | input=input->next; |
---|
467 | } |
---|
468 | |
---|
469 | #ifdef HPC_DEBUG |
---|
470 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
471 | #endif |
---|
472 | #ifdef USE_CALLBACK |
---|
473 | invokeCallback(m,inputs,NULL,1,1); |
---|
474 | invokeCallback(m,inputs,NULL,2,0); |
---|
475 | #endif |
---|
476 | if(getMapFromMaps(m,"lenv","mapError")!=NULL){ |
---|
477 | #ifdef USE_CALLBACK |
---|
478 | invokeCallback(*main_conf,inputs,NULL,7,0); |
---|
479 | #endif |
---|
480 | return -1; |
---|
481 | } |
---|
482 | #ifdef HPC_DEBUG |
---|
483 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
484 | dumpMaps(inputs); |
---|
485 | #endif |
---|
486 | |
---|
487 | // Upload data on HPC |
---|
488 | if(runUpload(main_conf)==false){ |
---|
489 | errorException (*main_conf, _("Unable to lock the file for upload!"), |
---|
490 | "InternalError", NULL); |
---|
491 | #ifdef HPC_DEBUG |
---|
492 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
493 | #endif |
---|
494 | #ifdef USE_CALLBACK |
---|
495 | invokeCallback(*main_conf,inputs,NULL,7,0); |
---|
496 | #endif |
---|
497 | #ifdef HPC_DEBUG |
---|
498 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
499 | #endif |
---|
500 | return -1; |
---|
501 | } |
---|
502 | #ifdef HPC_DEBUG |
---|
503 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
504 | #endif |
---|
505 | #ifdef USE_CALLBACK |
---|
506 | invokeCallback(m,inputs,NULL,2,1); |
---|
507 | #endif |
---|
508 | #ifdef HPC_DEBUG |
---|
509 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
510 | #endif |
---|
511 | |
---|
512 | // Add the filename to generate for every output to parameters |
---|
513 | input=*real_outputs; |
---|
514 | #ifdef HPC_DEBUG |
---|
515 | dumpMaps(input); |
---|
516 | #endif |
---|
517 | while(input!=NULL){ |
---|
518 | // Parse all outputs including inner outputs if required. |
---|
519 | if(input->child==NULL){ |
---|
520 | // Name every files that should be produced by the service execution |
---|
521 | map* mime=getMap(input->content,"mimeType"); |
---|
522 | char* targetName; |
---|
523 | if(mime!=NULL){ |
---|
524 | bool hasExt=false; |
---|
525 | map* fileExt=getFileExtensionMap(mime->value,&hasExt); |
---|
526 | targetName=(char*)malloc((strlen(s->name)+strlen(input->name)+strlen(uuid->value)+strlen(fileExt->value)+11)*sizeof(char)); |
---|
527 | sprintf(targetName,"output_%s_%s_%s.%s",s->name,input->name,uuid->value,fileExt->value); |
---|
528 | freeMap(&fileExt); |
---|
529 | free(fileExt); |
---|
530 | }else{ |
---|
531 | targetName=(char*)malloc((strlen(s->name)+strlen(input->name)+strlen(uuid->value)+14)*sizeof(char)); |
---|
532 | sprintf(targetName,"output_%s_%s_%s.tif",s->name,input->name,uuid->value); |
---|
533 | } |
---|
534 | char *targetPath=(char*)malloc((strlen(targetPathMap->value)+strlen(targetName)+2)*sizeof(char)); |
---|
535 | sprintf(targetPath,"%s/%s",targetPathMap->value,targetName); |
---|
536 | map *tmpUrl=getMapFromMaps(*main_conf,"main","tmpUrl"); |
---|
537 | char *targetUrl=(char*)malloc((strlen(tmpUrl->value)+strlen(targetName)+2)*sizeof(char)); |
---|
538 | sprintf(targetUrl,"%s/%s",tmpUrl->value,targetName); |
---|
539 | free(targetName); |
---|
540 | setMapInMaps(*real_outputs,input->name,"generated_file",targetPath); |
---|
541 | addToMap(input->content,"generated_url",targetUrl); |
---|
542 | free(targetUrl); |
---|
543 | { |
---|
544 | parameters_cnt+=1; |
---|
545 | if(parameters_cnt==1) |
---|
546 | parameters=(char**)malloc(parameters_cnt*sizeof(char*)); |
---|
547 | else |
---|
548 | parameters=(char**)realloc(parameters,parameters_cnt*sizeof(char*)); |
---|
549 | // We should verify if any optional tag for output is required |
---|
550 | // (i.e. -out output.tiff *int8*), meaning that we should search |
---|
551 | // for a corresponding inputs name. |
---|
552 | map* inValue=getMapFromMaps(*real_inputs,input->name,"value"); |
---|
553 | if(inValue!=NULL){ |
---|
554 | parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(targetPath)+strlen(inValue->value)+4)*sizeof(char)); |
---|
555 | sprintf(parameters[parameters_cnt-1],"-%s %s %s",input->name,targetPath,inValue->value); |
---|
556 | }else{ |
---|
557 | parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(targetPath)+3)*sizeof(char)); |
---|
558 | sprintf(parameters[parameters_cnt-1],"-%s %s",input->name,targetPath); |
---|
559 | } |
---|
560 | } |
---|
561 | free(targetPath); |
---|
562 | }// In other case it means we need to return the cache_file as generated_file |
---|
563 | else{ |
---|
564 | // Name every files that should be produced by the service execution |
---|
565 | map* mime=getMap(input->child->content,"mimeType"); |
---|
566 | char* targetName; |
---|
567 | if(mime!=NULL){ |
---|
568 | bool hasExt=false; |
---|
569 | map* fileExt=getFileExtensionMap(mime->value,&hasExt); |
---|
570 | targetName=(char*)malloc((strlen(s->name)+strlen(input->name)+strlen(uuid->value)+strlen(fileExt->value)+11)*sizeof(char)); |
---|
571 | sprintf(targetName,"output_%s_%s_%s.%s",s->name,input->name,uuid->value,fileExt->value); |
---|
572 | freeMap(&fileExt); |
---|
573 | free(fileExt); |
---|
574 | }else{ |
---|
575 | targetName=(char*)malloc((strlen(s->name)+strlen(input->name)+strlen(uuid->value)+14)*sizeof(char)); |
---|
576 | sprintf(targetName,"output_%s_%s_%s.tif",s->name,input->name,uuid->value); |
---|
577 | } |
---|
578 | char *targetPath=(char*)malloc((strlen(targetPathMap->value)+strlen(targetName)+2)*sizeof(char)); |
---|
579 | sprintf(targetPath,"%s/%s",targetPathMap->value,targetName); |
---|
580 | map *tmpUrl=getMapFromMaps(*main_conf,"main","tmpUrl"); |
---|
581 | char *targetUrl=(char*)malloc((strlen(tmpUrl->value)+strlen(targetName)+2)*sizeof(char)); |
---|
582 | sprintf(targetUrl,"%s/%s",tmpUrl->value,targetName); |
---|
583 | free(targetName); |
---|
584 | addToMap(input->content,"generated_file",targetPath); |
---|
585 | addToMap(input->content,"storage",targetPath); |
---|
586 | addToMap(input->content,"generated_url",targetUrl); |
---|
587 | free(targetUrl); |
---|
588 | if(strcasecmp(input->name,"wms_link")!=0&& |
---|
589 | strcasecmp(input->name,"wcs_link")!=0 && |
---|
590 | strcasecmp(input->name,"wfs_link")!=0){ |
---|
591 | parameters_cnt+=1; |
---|
592 | if(parameters_cnt==1) |
---|
593 | parameters=(char**)malloc(parameters_cnt*sizeof(char*)); |
---|
594 | else |
---|
595 | parameters=(char**)realloc(parameters,parameters_cnt*sizeof(char*)); |
---|
596 | // We should verify if any optional tag for output is required |
---|
597 | // (i.e. -out output.tiff *int8*), meaning that we should search |
---|
598 | // for a corresponding inputs name. |
---|
599 | map* inValue=getMapFromMaps(*real_inputs,input->name,"value"); |
---|
600 | if(inValue!=NULL){ |
---|
601 | parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(targetPath)+strlen(inValue->value)+4)*sizeof(char)); |
---|
602 | sprintf(parameters[parameters_cnt-1],"-%s %s %s",input->name,targetPath,inValue->value); |
---|
603 | }else{ |
---|
604 | parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(targetPath)+3)*sizeof(char)); |
---|
605 | sprintf(parameters[parameters_cnt-1],"-%s %s",input->name,targetPath); |
---|
606 | } |
---|
607 | } |
---|
608 | free(targetPath); |
---|
609 | } |
---|
610 | input=input->next; |
---|
611 | } |
---|
612 | // Produce the SBATCH File locally |
---|
613 | char *scriptPath=(char*)malloc((strlen(s->name)+strlen(tmpPath->value)+strlen(uuid->value)+10)*sizeof(char)); |
---|
614 | sprintf(scriptPath,"%s/zoo_%s_%s.sh",tmpPath->value,s->name,uuid->value); |
---|
615 | setMapInMaps(*main_conf,"lenv","local_script",scriptPath); |
---|
616 | #ifdef HPC_DEBUG |
---|
617 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
618 | fflush(stderr); |
---|
619 | #endif |
---|
620 | #ifdef USE_CALLBACK |
---|
621 | invokeCallback(m,inputs,NULL,3,0); |
---|
622 | #endif |
---|
623 | #ifdef HPC_DEBUG |
---|
624 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
625 | fflush(stderr); |
---|
626 | #endif |
---|
627 | FILE* scriptFile=fopen(scriptPath,"w+"); |
---|
628 | map* headerMap=getMapFromMaps(*main_conf,configurationId,"jobscript_header"); |
---|
629 | if(headerMap!=NULL){ |
---|
630 | // Use the header file if defined in the HPC section of the main.cfg file |
---|
631 | struct stat f_status; |
---|
632 | int s=stat(headerMap->value, &f_status); |
---|
633 | if(s==0){ |
---|
634 | char* fcontent=(char*)malloc(sizeof(char)*(f_status.st_size+1)); |
---|
635 | FILE* f=fopen(headerMap->value,"rb"); |
---|
636 | fread(fcontent,f_status.st_size,1,f); |
---|
637 | int fsize=f_status.st_size; |
---|
638 | fcontent[fsize]=0; |
---|
639 | fclose(f); |
---|
640 | fprintf(scriptFile,"%s\n### --- ZOO-Service HEADER end --- ###\n\n",fcontent); |
---|
641 | free(fcontent); |
---|
642 | }else |
---|
643 | fprintf(scriptFile,"#!/bin/bash\n\n### *** Default ZOO-Service HEADER (no header found) *** ###\n\n"); |
---|
644 | }else |
---|
645 | fprintf(scriptFile,"#!/bin/bash\n\n### *** Default ZOO-Service HEADER *** ###\n\n"); |
---|
646 | maps* hpc_opts=getMaps(*main_conf,configurationId); |
---|
647 | if(hpc_opts!=NULL){ |
---|
648 | map* hpc_opts_content=hpc_opts->content; |
---|
649 | while(hpc_opts_content!=NULL){ |
---|
650 | if(strncasecmp(hpc_opts_content->name,"sbatch_options_",15)==0) |
---|
651 | fprintf(scriptFile,"#SBATCH --%s=%s\n",strstr(hpc_opts_content->name,"sbatch_options_")+15,hpc_opts_content->value); |
---|
652 | hpc_opts_content=hpc_opts_content->next; |
---|
653 | } |
---|
654 | } |
---|
655 | fprintf(scriptFile,"#SBATCH --job-name=ZOO-Project_%s_%s\n\n",uuid->value,s->name); |
---|
656 | map* mods=getMap(s->content,"hpcModules"); |
---|
657 | if(mods!=NULL) |
---|
658 | fprintf(scriptFile,"#SBATCH --export=MODULES=%s\n",mods->value); |
---|
659 | |
---|
660 | map* bodyMap=getMapFromMaps(*main_conf,configurationId,"jobscript_body"); |
---|
661 | if(bodyMap!=NULL){ |
---|
662 | // Use the header file if defined in the HPC section of the main.cfg file |
---|
663 | struct stat f_status; |
---|
664 | int s=stat(bodyMap->value, &f_status); |
---|
665 | if(s==0){ |
---|
666 | char* fcontent=(char*)malloc(sizeof(char)*(f_status.st_size+1)); |
---|
667 | FILE* f=fopen(bodyMap->value,"rb"); |
---|
668 | fread(fcontent,f_status.st_size,1,f); |
---|
669 | int fsize=f_status.st_size; |
---|
670 | fcontent[fsize]=0; |
---|
671 | fclose(f); |
---|
672 | fprintf(scriptFile,"%s\n### --- ZOO-Service BODY end --- ###\n\n",fcontent); |
---|
673 | free(fcontent); |
---|
674 | }else |
---|
675 | fprintf(scriptFile,"\n### *** Default ZOO-Service BODY (no body found) *** ###\n\n"); |
---|
676 | }else |
---|
677 | fprintf(scriptFile,"\n### *** Default ZOO-Service BODY *** ###\n\n"); |
---|
678 | |
---|
679 | map* sp=getMap(s->content,"serviceProvider"); |
---|
680 | |
---|
681 | // Require to produce the command line to be executed |
---|
682 | fprintf(scriptFile,"\n\necho \"Job started at: $(date)\"\n"); |
---|
683 | fprintf(scriptFile,"echo \"Running service: [%s]\"\n",sp->value); |
---|
684 | fprintf(scriptFile,"%s ",sp->value); |
---|
685 | for(int i=0;i<parameters_cnt;i++){ |
---|
686 | fprintf(scriptFile," %s",parameters[i]); |
---|
687 | } |
---|
688 | for(int i=parameters_cnt-1;i>=0;i--){ |
---|
689 | free(parameters[i]); |
---|
690 | } |
---|
691 | free(parameters); |
---|
692 | fprintf(scriptFile,"\n"); |
---|
693 | fprintf(scriptFile,"echo \"Job finished at: $(date)\"\n"); |
---|
694 | map* footerMap=getMapFromMaps(*main_conf,configurationId,"jobscript_footer"); |
---|
695 | if(footerMap!=NULL){ |
---|
696 | // Use the footer file if defined in the HPC section of the main.cfg file |
---|
697 | struct stat f_status; |
---|
698 | int s=stat(footerMap->value, &f_status); |
---|
699 | if(s==0){ |
---|
700 | char* fcontent=(char*)malloc(sizeof(char)*(f_status.st_size+1)); |
---|
701 | FILE* f=fopen(footerMap->value,"rb"); |
---|
702 | fread(fcontent,f_status.st_size,1,f); |
---|
703 | int fsize=f_status.st_size; |
---|
704 | fcontent[fsize]=0; |
---|
705 | fclose(f); |
---|
706 | char* ffcontent=(char*)malloc((strlen(fcontent)+(3*strlen(uuid->value))+1)*sizeof(char)); |
---|
707 | sprintf(ffcontent,fcontent,uuid->value,uuid->value,uuid->value); |
---|
708 | fprintf(scriptFile,"%s\n### --- ZOO-Service FOOTER end --- ###\n\n",ffcontent); |
---|
709 | free(fcontent); |
---|
710 | }else |
---|
711 | fprintf(scriptFile,"### *** Default ZOO-Service FOOTER (footer file failed to load) *** ###\n\n"); |
---|
712 | }else |
---|
713 | fprintf(scriptFile,"### *** Default ZOO-Service FOOTER (no footer found) *** ###\n\n"); |
---|
714 | fflush(scriptFile); |
---|
715 | fclose(scriptFile); |
---|
716 | #ifdef HPC_DEBUG |
---|
717 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
718 | #endif |
---|
719 | #ifdef USE_CALLBACK |
---|
720 | invokeCallback(m,inputs,NULL,3,1); |
---|
721 | #endif |
---|
722 | #ifdef HPC_DEBUG |
---|
723 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
724 | #endif |
---|
725 | |
---|
726 | // Upload the SBATCH File to the remote host |
---|
727 | #ifdef HPC_DEBUG |
---|
728 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
729 | #endif |
---|
730 | #ifdef USE_CALLBACK |
---|
731 | invokeCallback(m,inputs,NULL,4,0); |
---|
732 | #endif |
---|
733 | #ifdef HPC_DEBUG |
---|
734 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
735 | #endif |
---|
736 | targetPathMap=getMapFromMaps(*main_conf,configurationId,"remote_work_path"); |
---|
737 | if(targetPathMap==NULL){ |
---|
738 | setMapInMaps(*main_conf,"lenv","message",_("There is no remote_work_path defined in your section!")); |
---|
739 | setMapInMaps(*main_conf,"lenv","status","failed"); |
---|
740 | errorException (*main_conf, _("There is no remote_work_path defined in your section!"), |
---|
741 | "InternalError", NULL); |
---|
742 | #ifdef HPC_DEBUG |
---|
743 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
744 | fflush(stderr); |
---|
745 | #endif |
---|
746 | #ifdef USE_CALLBACK |
---|
747 | invokeCallback(*main_conf,NULL,NULL,7,0); |
---|
748 | #endif |
---|
749 | #ifdef HPC_DEBUG |
---|
750 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
751 | fflush(stderr); |
---|
752 | #endif |
---|
753 | return SERVICE_FAILED; |
---|
754 | } |
---|
755 | char* targetName=strrchr(scriptPath,'/'); |
---|
756 | char *targetPath=(char*)malloc((strlen(targetPathMap->value)+strlen(targetName)+2)*sizeof(char)); |
---|
757 | sprintf(targetPath,"%s/%s",targetPathMap->value,targetName); |
---|
758 | setMapInMaps(*main_conf,"lenv","remote_script",targetPath); |
---|
759 | SSHCON *test=ssh_connect(*main_conf); |
---|
760 | int copy0=ssh_copy(*main_conf,scriptPath,targetPath,ssh_get_cnt(*main_conf)); |
---|
761 | unlink(scriptPath); |
---|
762 | free(scriptPath); |
---|
763 | if(copy0!=true){ |
---|
764 | setMapInMaps(*main_conf,"lenv","message",_("Unable to upload the script")); |
---|
765 | #ifdef USE_CALLBACK |
---|
766 | invokeCallback(*main_conf,NULL,NULL,7,0); |
---|
767 | #endif |
---|
768 | errorException(*main_conf,_("Unable to upload the script"),"NoApplicableCode",NULL); |
---|
769 | return -1; |
---|
770 | } |
---|
771 | // Execute the SBATCH script remotely |
---|
772 | addReadLocks(main_conf); |
---|
773 | map* subStr=getMapFromMaps(*main_conf,configurationId,"sbatch_substr"); |
---|
774 | char *command=(char*)malloc((strlen(targetPath)+strlen(targetPathMap->value)+strlen(subStr->value)+strlen(uuid->value)+137)*sizeof(char)); |
---|
775 | sprintf(command,"sbatch %s 2> %s/error_%s.log | sed \"s:%s::g\"",targetPath,targetPathMap->value,uuid->value,subStr->value); |
---|
776 | if(ssh_exec(*main_conf,command,ssh_get_cnt(m))<=0){ |
---|
777 | // The sbatch command has failed! |
---|
778 | // Download the error log file from the HPC server |
---|
779 | char tmpS[1024]; |
---|
780 | free(command); |
---|
781 | command=(char*)malloc((strlen(targetPathMap->value)+strlen(uuid->value)+11)*sizeof(char)); |
---|
782 | sprintf(command,"%s/error_%s.log",targetPathMap->value,uuid->value); |
---|
783 | targetName=strrchr(command,'/'); |
---|
784 | free(targetPath); |
---|
785 | targetPath=(char*)malloc((strlen(tmpPath->value)+strlen(targetName)+2)*sizeof(char)); |
---|
786 | sprintf(targetPath,"%s/%s",tmpPath->value,targetName); |
---|
787 | if(ssh_fetch(*main_conf,targetPath,command,ssh_get_cnt(m))==0){ |
---|
788 | struct stat f_status; |
---|
789 | int ts=stat(targetPath, &f_status); |
---|
790 | if(ts==0) { |
---|
791 | char* fcontent = NULL; |
---|
792 | fcontent=(char*)malloc(sizeof(char)*(f_status.st_size+1)); |
---|
793 | FILE* f=fopen(targetPath,"rb"); |
---|
794 | fread(fcontent,f_status.st_size,1,f); |
---|
795 | int fsize=f_status.st_size; |
---|
796 | fcontent[fsize]=0; |
---|
797 | fclose(f); |
---|
798 | setMapInMaps(*main_conf,"lenv","message",fcontent); |
---|
799 | free(fcontent); |
---|
800 | }else |
---|
801 | setMapInMaps(*main_conf,"lenv","message",_("No message provided")); |
---|
802 | }else |
---|
803 | setMapInMaps(*main_conf,"lenv","message",_("Unable to fetch the remote error log file")); |
---|
804 | tmpPath=getMapFromMaps(m,"lenv","message"); |
---|
805 | #ifdef HPC_DEBUG |
---|
806 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
807 | fflush(stderr); |
---|
808 | #endif |
---|
809 | #ifdef USE_CALLBACK |
---|
810 | invokeCallback(*main_conf,NULL,NULL,7,0); |
---|
811 | #endif |
---|
812 | #ifdef HPC_DEBUG |
---|
813 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
814 | fflush(stderr); |
---|
815 | #endif |
---|
816 | sprintf(tmpS, "Cannot execute the HPC ZOO-Service %s using %s: %s", s->name, configurationId, tmpPath->value); |
---|
817 | errorException(*main_conf,tmpS,"NoApplicableCode",NULL); |
---|
818 | free(command); |
---|
819 | free(targetPath); |
---|
820 | ssh_close(*main_conf); |
---|
821 | removeReadLocks(main_conf); |
---|
822 | return -1; |
---|
823 | } |
---|
824 | free(targetPath); |
---|
825 | #ifdef HPC_DEBUG |
---|
826 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
827 | fflush(stderr); |
---|
828 | #endif |
---|
829 | #ifdef USE_CALLBACK |
---|
830 | invokeCallback(m,NULL,NULL,4,1); |
---|
831 | #endif |
---|
832 | #ifdef HPC_DEBUG |
---|
833 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
834 | fflush(stderr); |
---|
835 | #endif |
---|
836 | free(command); |
---|
837 | #ifdef HPC_DEBUG |
---|
838 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
839 | fflush(stderr); |
---|
840 | #endif |
---|
841 | |
---|
842 | struct sockaddr_un addr; |
---|
843 | memset(&addr, 0, sizeof(addr)); |
---|
844 | addr.sun_family = AF_UNIX; |
---|
845 | int rc, cl, fd = socket(AF_UNIX, SOCK_STREAM, 0); |
---|
846 | char *sname=(char*)malloc((strlen(tmpPath->value)+strlen(uuid->value)+20)); |
---|
847 | sprintf(sname,"%s/.wait_socket_%s.sock",tmpPath->value,uuid->value); |
---|
848 | strncpy(addr.sun_path, sname, sizeof(addr.sun_path)-1); |
---|
849 | |
---|
850 | if (bind(fd, (struct sockaddr*)&addr, sizeof(addr)) == -1) { |
---|
851 | perror("bind error"); |
---|
852 | setMapInMaps(*main_conf,"lenv","message",_("Unable to bind socket!")); |
---|
853 | errorException (*main_conf, _("Unable to bind socket!"), |
---|
854 | "InternalError", NULL); |
---|
855 | #ifdef HPC_DEBUG |
---|
856 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
857 | fflush(stderr); |
---|
858 | #endif |
---|
859 | #ifdef USE_CALLBACK |
---|
860 | invokeCallback(*main_conf,NULL,NULL,7,0); |
---|
861 | #endif |
---|
862 | removeReadLocks(main_conf); |
---|
863 | #ifdef HPC_DEBUG |
---|
864 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
865 | fflush(stderr); |
---|
866 | #endif |
---|
867 | return -1; |
---|
868 | } |
---|
869 | #ifdef HPC_DEBUG |
---|
870 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
871 | fflush(stderr); |
---|
872 | #endif |
---|
873 | if (listen(fd, 5) == -1) { |
---|
874 | setMapInMaps(*main_conf,"lenv","message",_("Listen error")); |
---|
875 | errorException (*main_conf, _("Listen error"), |
---|
876 | "InternalError", NULL); |
---|
877 | #ifdef HPC_DEBUG |
---|
878 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
879 | fflush(stderr); |
---|
880 | #endif |
---|
881 | #ifdef USE_CALLBACK |
---|
882 | invokeCallback(*main_conf,NULL,NULL,7,0); |
---|
883 | #endif |
---|
884 | removeReadLocks(main_conf); |
---|
885 | #ifdef HPC_DEBUG |
---|
886 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
887 | fflush(stderr); |
---|
888 | #endif |
---|
889 | return -1; |
---|
890 | } |
---|
891 | if ( (cl = accept(fd, NULL, NULL)) == -1) { |
---|
892 | setMapInMaps(*main_conf,"lenv","message",_("Accept error")); |
---|
893 | errorException (*main_conf, _("Accept error"), |
---|
894 | "InternalError", NULL); |
---|
895 | #ifdef HPC_DEBUG |
---|
896 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
897 | fflush(stderr); |
---|
898 | #endif |
---|
899 | #ifdef USE_CALLBACK |
---|
900 | invokeCallback(*main_conf,NULL,NULL,7,0); |
---|
901 | #endif |
---|
902 | removeReadLocks(main_conf); |
---|
903 | #ifdef HPC_DEBUG |
---|
904 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
905 | fflush(stderr); |
---|
906 | #endif |
---|
907 | return -1; |
---|
908 | }else{ |
---|
909 | #ifdef HPC_DEBUG |
---|
910 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
911 | fflush(stderr); |
---|
912 | #endif |
---|
913 | int hasPassed=-1; |
---|
914 | char buf[11]; |
---|
915 | memset(&buf,0,11); |
---|
916 | while ( (rc=read(cl,buf,10)) ) { |
---|
917 | if(rc==0){ |
---|
918 | setMapInMaps(*main_conf,"lenv","message",_("Read closed")); |
---|
919 | errorException (*main_conf, _("Read closed"), |
---|
920 | "InternalError", NULL); |
---|
921 | #ifdef HPC_DEBUG |
---|
922 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
923 | fflush(stderr); |
---|
924 | #endif |
---|
925 | #ifdef USE_CALLBACK |
---|
926 | invokeCallback(*main_conf,NULL,NULL,7,0); |
---|
927 | #endif |
---|
928 | removeReadLocks(main_conf); |
---|
929 | #ifdef HPC_DEBUG |
---|
930 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
931 | fflush(stderr); |
---|
932 | #endif |
---|
933 | return -1; |
---|
934 | }else{ |
---|
935 | if(rc<0){ |
---|
936 | setMapInMaps(*main_conf,"lenv","message",_("Read error")); |
---|
937 | errorException (*main_conf, _("Read error"), |
---|
938 | "InternalError", NULL); |
---|
939 | #ifdef HPC_DEBUG |
---|
940 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
941 | fflush(stderr); |
---|
942 | #endif |
---|
943 | #ifdef USE_CALLBACK |
---|
944 | invokeCallback(*main_conf,NULL,NULL,7,0); |
---|
945 | #endif |
---|
946 | removeReadLocks(main_conf); |
---|
947 | #ifdef HPC_DEBUG |
---|
948 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
949 | fflush(stderr); |
---|
950 | #endif |
---|
951 | return -1; |
---|
952 | } |
---|
953 | } |
---|
954 | hasPassed=1; |
---|
955 | res=atoi(buf); |
---|
956 | unlink(sname); |
---|
957 | free(sname); |
---|
958 | removeReadLocks(main_conf); |
---|
959 | |
---|
960 | if(res==3){ |
---|
961 | #ifdef HPC_DEBUG |
---|
962 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
963 | fflush(stderr); |
---|
964 | #endif |
---|
965 | #ifdef USE_CALLBACK |
---|
966 | invokeCallback(m,NULL,outputs,5,0); |
---|
967 | #endif |
---|
968 | #ifdef HPC_DEBUG |
---|
969 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
970 | fflush(stderr); |
---|
971 | #endif |
---|
972 | |
---|
973 | // Read informations provided by FinalizeHPC as a configuration file |
---|
974 | // then, remove the file. |
---|
975 | map* jobid=getMapFromMaps(*main_conf,"lenv","usid"); |
---|
976 | map* tmpPath=getMapFromMaps(*main_conf,"main","tmpPath"); |
---|
977 | char *filePath=(char*)malloc((strlen(tmpPath->value)+strlen(jobid->value)+15)*sizeof(char)); |
---|
978 | sprintf(filePath,"%s/exec_status_%s",tmpPath->value,jobid->value); |
---|
979 | maps* lm = (maps *) malloc (MAPS_SIZE); |
---|
980 | lm->child=NULL; |
---|
981 | lm->next=NULL; |
---|
982 | int saved_stdout = dup (fileno (stdout)); |
---|
983 | dup2 (fileno (stderr), fileno (stdout)); |
---|
984 | conf_read(filePath,lm); |
---|
985 | //dumpMaps(m); |
---|
986 | fflush(stdout); |
---|
987 | dup2 (saved_stdout, fileno (stdout)); |
---|
988 | close(saved_stdout); |
---|
989 | unlink(filePath); |
---|
990 | free(filePath); |
---|
991 | addMapsToMaps(main_conf,lm); |
---|
992 | freeMaps(&lm); |
---|
993 | free(lm); |
---|
994 | |
---|
995 | input=*real_outputs; |
---|
996 | while(input!=NULL){ |
---|
997 | if(input->child==NULL){ |
---|
998 | map* generatedFile=getMap(input->content,"generated_file"); |
---|
999 | if(generatedFile!=NULL){ |
---|
1000 | char* filename=strrchr(generatedFile->value,'/'); |
---|
1001 | char* targetPath=(char*)malloc((strlen(tmpPath->value)+strlen(filename)+2)*sizeof(char)); |
---|
1002 | sprintf(targetPath,"%s/%s",tmpPath->value,filename); |
---|
1003 | test=ssh_connect(*main_conf); |
---|
1004 | if(ssh_fetch(*main_conf,targetPath,generatedFile->value,ssh_get_cnt(m))==0){ |
---|
1005 | setMapInMaps(*real_outputs,input->name,"generated_file",targetPath); |
---|
1006 | free(targetPath); |
---|
1007 | }else{ |
---|
1008 | map* hpcStdErr=getMapFromMaps(*main_conf,"henv","StdErr"); |
---|
1009 | // Added for using sacct in place of scontrol |
---|
1010 | char *sourcePath=NULL; |
---|
1011 | if(hpcStdErr!=NULL){ |
---|
1012 | sourcePath=(char*)malloc((strlen(targetPathMap->value)+strlen(hpcStdErr->value)+2)*sizeof(char)); |
---|
1013 | sprintf(sourcePath,"%s/%s",targetPathMap->value,hpcStdErr->value); |
---|
1014 | } |
---|
1015 | if(hpcStdErr!=NULL && sourcePath!=NULL && ssh_fetch(*main_conf,targetPath,sourcePath,ssh_get_cnt(m))==0){ |
---|
1016 | free(sourcePath); |
---|
1017 | struct stat f_status; |
---|
1018 | int ts=stat(targetPath, &f_status); |
---|
1019 | if(ts==0) { |
---|
1020 | char* fcontent = NULL; |
---|
1021 | fcontent=(char*)malloc(sizeof(char)*(f_status.st_size+1)); |
---|
1022 | FILE* f=fopen(targetPath,"rb"); |
---|
1023 | fread(fcontent,f_status.st_size,1,f); |
---|
1024 | int fsize=f_status.st_size; |
---|
1025 | fcontent[fsize]=0; |
---|
1026 | fclose(f); |
---|
1027 | setMapInMaps(*main_conf,"lenv","message",fcontent); |
---|
1028 | free(fcontent); |
---|
1029 | }else{ |
---|
1030 | char *tmpStr=(char*)malloc((strlen(targetPath)+strlen(_("Unable to fetch the remote file for %s"))+1)*sizeof(char)); |
---|
1031 | sprintf(tmpStr,_("Unable to fetch the remote file for %s"),targetPath); |
---|
1032 | setMapInMaps(*main_conf,"lenv","message",tmpStr); |
---|
1033 | free(tmpStr); |
---|
1034 | } |
---|
1035 | }else{ |
---|
1036 | char *tmpStr=(char*)malloc((strlen(filename)+strlen(_("Unable to fetch the remote file for %s"))+1)*sizeof(char)); |
---|
1037 | sprintf(tmpStr,_("Unable to fetch the remote file for %s"),filename); |
---|
1038 | setMapInMaps(*main_conf,"lenv","message",tmpStr); |
---|
1039 | free(tmpStr); |
---|
1040 | } |
---|
1041 | #ifdef USE_CALLBACK |
---|
1042 | invokeCallback(*main_conf,NULL,NULL,7,0); |
---|
1043 | #endif |
---|
1044 | return SERVICE_FAILED; |
---|
1045 | } |
---|
1046 | } |
---|
1047 | }else{ |
---|
1048 | map* generatedFile=getMap(input->content,"generated_file"); |
---|
1049 | map* generatedUrl=getMap(input->content,"generated_url"); |
---|
1050 | if(generatedFile!=NULL){ |
---|
1051 | char* filename=strrchr(generatedFile->value,'/'); |
---|
1052 | char* targetPath=(char*)malloc((strlen(tmpPath->value)+strlen(filename)+2)*sizeof(char)); |
---|
1053 | sprintf(targetPath,"%s/%s",tmpPath->value,filename); |
---|
1054 | test=ssh_connect(*main_conf); |
---|
1055 | if(ssh_fetch(*main_conf,targetPath,generatedFile->value,ssh_get_cnt(m))==0){ |
---|
1056 | maps* tmp=getMaps(*real_outputs,input->name); |
---|
1057 | char serviceName[9]; |
---|
1058 | freeMap(&tmp->content); |
---|
1059 | free(tmp->content); |
---|
1060 | tmp->content=NULL; |
---|
1061 | maps* output=getMaps(*real_outputs,input->name); |
---|
1062 | setMapInMaps(output->child,"download_link","generated_file",targetPath); |
---|
1063 | setMapInMaps(output->child,"download_link","generated_url",generatedUrl->value); |
---|
1064 | setMapInMaps(output->child,"download_link","storage",targetPath); |
---|
1065 | setMapInMaps(output->child,"download_link","useMapserver","false"); |
---|
1066 | setMapInMaps(output->child,"download_link","replicateStorageNext","true"); |
---|
1067 | setMapInMaps(output->child,"download_link","asReference","true"); |
---|
1068 | setMapInMaps(output->child,"download_link","inRequest","true"); |
---|
1069 | setMapInMaps(output->child,"wms_link","generated_file",targetPath); |
---|
1070 | setMapInMaps(output->child,"wms_link","storage",targetPath); |
---|
1071 | setMapInMaps(output->child,"wms_link","useMapserver","true"); |
---|
1072 | setMapInMaps(output->child,"wms_link","msOgc","WMS"); |
---|
1073 | setMapInMaps(output->child,"wms_link","requestedMimeType","image/png"); |
---|
1074 | setMapInMaps(output->child,"wms_link","asReference","true"); |
---|
1075 | if(getMaps(output->child,"wcs_link")!=NULL){ |
---|
1076 | sprintf(serviceName,"wcs_link"); |
---|
1077 | setMapInMaps(output->child,"wcs_link","msOgc","WCS"); |
---|
1078 | }else{ |
---|
1079 | sprintf(serviceName,"wfs_link"); |
---|
1080 | setMapInMaps(output->child,"wfs_link","msOgc","WFS"); |
---|
1081 | } |
---|
1082 | setMapInMaps(output->child,serviceName,"storage",targetPath); |
---|
1083 | setMapInMaps(output->child,serviceName,"generated_file",targetPath); |
---|
1084 | setMapInMaps(output->child,serviceName,"useMapserver","true"); |
---|
1085 | setMapInMaps(output->child,serviceName,"asReference","true"); |
---|
1086 | }else{ |
---|
1087 | map* hpcStdErr=getMapFromMaps(*main_conf,"henv","StdErr"); |
---|
1088 | char *sourcePath=NULL; |
---|
1089 | if(hpcStdErr!=NULL){ |
---|
1090 | dumpMap(hpcStdErr); |
---|
1091 | sourcePath=(char*)malloc((strlen(targetPathMap->value)+strlen(hpcStdErr->value)+2)*sizeof(char)); |
---|
1092 | sprintf(sourcePath,"%s/%s",targetPathMap->value,hpcStdErr->value); |
---|
1093 | } |
---|
1094 | if(hpcStdErr!=NULL && sourcePath!=NULL && ssh_fetch(*main_conf,targetPath,sourcePath,ssh_get_cnt(m))==0){ |
---|
1095 | free(sourcePath); |
---|
1096 | struct stat f_status; |
---|
1097 | int ts=stat(targetPath, &f_status); |
---|
1098 | if(ts==0) { |
---|
1099 | char* fcontent = NULL; |
---|
1100 | fcontent=(char*)malloc(sizeof(char)*(f_status.st_size+1)); |
---|
1101 | FILE* f=fopen(targetPath,"rb"); |
---|
1102 | fread(fcontent,f_status.st_size,1,f); |
---|
1103 | int fsize=f_status.st_size; |
---|
1104 | fcontent[fsize]=0; |
---|
1105 | fclose(f); |
---|
1106 | setMapInMaps(*main_conf,"lenv","message",fcontent); |
---|
1107 | free(fcontent); |
---|
1108 | }else{ |
---|
1109 | char *tmpStr=(char*)malloc((strlen(targetPath)+strlen(_("Unable to fetch the remote file for %s"))+1)*sizeof(char)); |
---|
1110 | sprintf(tmpStr,_("Unable to fetch the remote file for %s"),targetPath); |
---|
1111 | setMapInMaps(*main_conf,"lenv","message",tmpStr); |
---|
1112 | free(tmpStr); |
---|
1113 | } |
---|
1114 | }else{ |
---|
1115 | char *tmpStr=(char*)malloc((strlen(sourcePath)+strlen(_("Unable to fetch the remote file for %s"))+1)*sizeof(char)); |
---|
1116 | sprintf(tmpStr,_("Unable to fetch the remote file for %s"),sourcePath); |
---|
1117 | setMapInMaps(*main_conf,"lenv","message",tmpStr); |
---|
1118 | free(tmpStr); |
---|
1119 | } |
---|
1120 | #ifdef USE_CALLBACK |
---|
1121 | invokeCallback(*main_conf,NULL,NULL,7,0); |
---|
1122 | #endif |
---|
1123 | return SERVICE_FAILED; |
---|
1124 | } |
---|
1125 | free(targetPath); |
---|
1126 | } |
---|
1127 | } |
---|
1128 | input=input->next; |
---|
1129 | } |
---|
1130 | |
---|
1131 | }else{ |
---|
1132 | // Try to access remotely to the log file and return a more relevant error message |
---|
1133 | setMapInMaps(*main_conf,"lenv","message",_("HPC Execution failed!")); |
---|
1134 | errorException (*main_conf, _("HPC Execution failed!"), |
---|
1135 | "InternalError", NULL); |
---|
1136 | #ifdef HPC_DEBUG |
---|
1137 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
1138 | fflush(stderr); |
---|
1139 | #endif |
---|
1140 | #ifdef USE_CALLBACK |
---|
1141 | invokeCallback(*main_conf,NULL,NULL,7,0); |
---|
1142 | #endif |
---|
1143 | #ifdef HPC_DEBUG |
---|
1144 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
1145 | fflush(stderr); |
---|
1146 | #endif |
---|
1147 | } |
---|
1148 | } |
---|
1149 | if(hasPassed<0){ |
---|
1150 | perror("Failed to read"); |
---|
1151 | setMapInMaps(*main_conf,"lenv","message",_("Unable to parse the value returned by remote execution")); |
---|
1152 | errorException (*main_conf, _("Unable to parse the value returned by remote execution"), |
---|
1153 | "InternalError", NULL); |
---|
1154 | #ifdef HPC_DEBUG |
---|
1155 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
1156 | fflush(stderr); |
---|
1157 | #endif |
---|
1158 | #ifdef USE_CALLBACK |
---|
1159 | invokeCallback(*main_conf,NULL,NULL,7,0); |
---|
1160 | #endif |
---|
1161 | #ifdef HPC_DEBUG |
---|
1162 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
1163 | fflush(stderr); |
---|
1164 | #endif |
---|
1165 | return SERVICE_FAILED; |
---|
1166 | } |
---|
1167 | } |
---|
1168 | #ifdef HPC_DEBUG |
---|
1169 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
1170 | fflush(stderr); |
---|
1171 | #endif |
---|
1172 | ssh_close(*main_conf); |
---|
1173 | #ifdef HPC_DEBUG |
---|
1174 | fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__); |
---|
1175 | fflush(stderr); |
---|
1176 | #endif |
---|
1177 | return res; |
---|
1178 | } |
---|