Ignore:
Timestamp:
Oct 11, 2018, 4:30:46 PM (6 years ago)
Author:
djay
Message:

Check for md5sum of any file in the cache to avoid sending the same file on HPC server twice. Add the multiple LiteralData? inputs support for HPC services. Remove condition for defining SCALE for every band in the outputed MapServer? mapfile.

Location:
branches/prototype-v0/zoo-project/zoo-kernel
Files:
6 edited

Legend:

Unmodified
Added
Removed
  • branches/prototype-v0/zoo-project/zoo-kernel/caching.c

    r880 r886  
    177177  fwrite(md5fstr,sizeof(char),strlen(md5fstr),fo);
    178178  free(md5fstr);
     179  free(storage);
    179180  fclose(fo);
    180181  return 0;
  • branches/prototype-v0/zoo-project/zoo-kernel/meta_sql.c

    r854 r886  
    4242#endif
    4343#define META_SERVICES_LIST_ALL \
    44   "select id,identifier,title,abstract,service_type,service_provider"\
     44  "select id,identifier,title,abstract,service_type,service_provider,conf_id"\
    4545  " from ows_process"
    46 #define META_SERVICES_LIST_ALL_LENGTH 83
     46#define META_SERVICES_LIST_ALL_LENGTH strlen(META_SERVICES_LIST_ALL)
    4747
    4848#define META_SERVICES_KEYWORDS_FROM_PROCESS \
     
    5050  " (SELECT keywords_id FROM CollectionDB.DescriptionsKeywordsAssignment"\
    5151  " where descriptions_id=%s) "
    52 #define META_SERVICES_KEYWORDS_FROM_PROCESS_LENGTH 162
     52#define META_SERVICES_KEYWORDS_FROM_PROCESS_LENGTH strlen(META_SERVICES_KEYWORDS_FROM_PROCESS)
    5353
    5454#define META_SERVICES_META_FROM_ANYTHING \
     
    5656  " (SELECT metadata_id FROM CollectionDB.DescriptionsMetadataAssignment"\
    5757  " where descriptions_id=%s) "
    58 #define META_SERVICES_META_FROM_ANYTHING_LENGTH 162
     58#define META_SERVICES_META_FROM_ANYTHING_LENGTH strlen(META_SERVICES_META_FROM_ANYTHING)
    5959
    6060#define META_SERVICES_AP_FROM_ANYTHING \
     
    6262  " (SELECT additional_parameters_id FROM CollectionDB.DescriptionsAdditionalParametersAssignment"\
    6363  " where descriptions_id=%s) "
    64 #define META_SERVICES_AP_FROM_ANYTHING_LENGTH 199
     64#define META_SERVICES_AP_FROM_ANYTHING_LENGTH strlen(META_SERVICES_AP_FROM_ANYTHING)
    6565
    6666#define META_SERVICES_AP_FROM_AP \
    6767  "SELECT key,value FROM CollectionDB.ows_AdditionalParameter where additional_parameters_id =$q$%s$q$"
    68 #define META_SERVICES_AP_FROM_AP_LENGTH 100
     68#define META_SERVICES_AP_FROM_AP_LENGTH strlen(META_SERVICES_AP_FROM_AP)
    6969
    7070#define META_SERVICES_LIST_INPUTS_FROM_PROCESS                          \
    7171  "select id, identifier,title,abstract,min_occurs,max_occurs from CollectionDB.ows_Input where id in (SELECT input_id from CollectionDB.ProcessInputAssignment where process_id=%s) order by id"
    72 #define META_SERVICES_LIST_INPUTS_FROM_PROCESS_LENGTH 190
     72#define META_SERVICES_LIST_INPUTS_FROM_PROCESS_LENGTH strlen(META_SERVICES_LIST_INPUTS_FROM_PROCESS)
    7373
    7474#define META_SERVICES_LIST_INPUTS_FROM_INPUT                            \
    7575  "select id, identifier,title,abstract,min_occurs,max_occurs from CollectionDB.ows_Input where id in (SELECT child_input from CollectionDB.InputInputAssignment where parent_input=%s) order by id"
    76 #define META_SERVICES_LIST_INPUTS_FROM_INPUT_LENGTH 193
     76#define META_SERVICES_LIST_INPUTS_FROM_INPUT_LENGTH strlen(META_SERVICES_LIST_INPUTS_FROM_INPUT)
    7777
    7878#define META_SERVICES_LIST_OUTPUTS_FROM_PROCESS \
    7979  "select id, identifier,title,abstract from CollectionDB.ows_Output where id in (SELECT output_id from CollectionDB.ProcessOutputAssignment where process_id=%s) order by id"
    80 #define META_SERVICES_LIST_OUTPUTS_FROM_PROCESS_LENGTH 171
     80#define META_SERVICES_LIST_OUTPUTS_FROM_PROCESS_LENGTH strlen(META_SERVICES_LIST_OUTPUTS_FROM_PROCESS)
    8181
    8282#define META_SERVICES_LIST_OUTPUTS_FROM_OUTPUT \
    8383  "select id, identifier,title,abstract from CollectionDB.ows_Output where id in (SELECT child_output from CollectionDB.OutputOutputAssignment where parent_output=%s) order by id"
    84 #define META_SERVICES_LIST_OUTPUTS_FROM_OUTPUT_LENGTH 176
     84#define META_SERVICES_LIST_OUTPUTS_FROM_OUTPUT_LENGTH strlen(META_SERVICES_LIST_OUTPUTS_FROM_OUTPUT)
    8585
    8686#define META_SERVICES_LIST_LITERAL_FROM_IO \
    8787  "select (SELECT name as type FROM CollectionDB.PrimitiveDatatypes where CollectionDB.PrimitiveDatatypes.id=data_type_id),default_value,(SELECT uom from CollectionDB.PrimitiveUOM where id=CollectionDB.LiteralDataDomain.uom),translate(translate(ARRAY((SELECT allowed_Value from CollectionDB.AllowedValues where id in (SELECT allowed_value_id from CollectionDB.AllowedValuesAssignment where literal_data_domain_id=CollectionDB.LiteralDataDomain.id)))::varchar,'{',''),'}',''),def as allowedvalues from CollectionDB.LiteralDataDomain where id in (SELECT data_description_id from CollectionDB.%sDataDescriptionAssignment where %s_id = %s);"
    88 #define META_SERVICES_LIST_LITERAL_FROM_IO_LENGTH 634
     88#define META_SERVICES_LIST_LITERAL_FROM_IO_LENGTH strlen(META_SERVICES_LIST_LITERAL_FROM_IO)
    8989
    9090#define META_SERVICES_LIST_FORMATS_FROM_IO \
    9191  "select mime_type,encoding,schema,maximum_megabytes,CASE WHEN use_mapserver THEN 'true' ELSE 'false' END, ms_styles, def from CollectionDB.ows_Format,CollectionDB.PrimitiveFormats where CollectionDB.ows_Format.primitive_format_id=CollectionDB.PrimitiveFormats.id and CollectionDB.ows_Format.id in (SELECT format_id from collectiondb.ows_datadescription where id in ( SELECT data_description_id from CollectionDB.%sDataDescriptionAssignment where %s_id = %s))"
    92 #define META_SERVICES_LIST_FORMATS_FROM_IO_LENGTH 458
     92#define META_SERVICES_LIST_FORMATS_FROM_IO_LENGTH strlen(META_SERVICES_LIST_FORMATS_FROM_IO)
    9393
    9494/**
     
    387387      addToMap(s->content,"serviceType",poFeature->GetFieldAsString( 4 ));
    388388      addToMap(s->content,"serviceProvider",poFeature->GetFieldAsString( 5 ));
     389      addToMap(s->content,"confId",poFeature->GetFieldAsString( 6 ));
    389390      addToMap(s->content,"fromDb","true");
    390391      s->metadata=NULL;
  • branches/prototype-v0/zoo-project/zoo-kernel/service_callback.c

    r877 r886  
    769769      char *keys[6][2]={
    770770        {
    771           "SubmitTime",
     771          //"SubmitTime",
     772          "Submit",
    772773          "hpc_submission_date"
    773774        },
     
    781782        },
    782783        {
    783           "StartTime",
     784          //"StartTime",
     785          "Start",
    784786          "hpc_start_date"
    785787        },
    786788        {
    787           "EndTime",
     789          //"EndTime",
     790          "End",
    788791          "hpc_end_date"
    789792        },
    790793        {
    791           "JobState",
     794          //"JobState",
     795          "State",
    792796          "hpc_status"
    793797        }       
  • branches/prototype-v0/zoo-project/zoo-kernel/service_internal_hpc.c

    r877 r886  
    299299  // then, set the configId key in the lenv section
    300300  char *serviceType;
    301   map* mServiceType=getMap(s->content,"serviceType");
     301  map* mServiceType=getMap(s->content,"confId");
    302302  if(mServiceType!=NULL)
    303303    serviceType=mServiceType->value;
     
    334334  while(input!=NULL && input->content!=NULL){
    335335    map* isInRequest=getMap(input->content,"inRequest");
    336     map* minNb=getMap(input->content,"minoccurs");
     336    map* minNb=getMap(input->content,"minOccurs");
    337337    if(getMaps(*real_outputs,input->name)==NULL &&
    338338       ( (isInRequest!=NULL && strncasecmp(isInRequest->value,"true",4)==0)
     
    397397        if(getMap(input->content,"dataType")!=NULL){
    398398          // For LitteralData, simply pass the value
    399           map* val=getMap(input->content,"value");
    400           parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(val->value)+3)*sizeof(char));
    401           sprintf(parameters[parameters_cnt-1],"-%s %s",input->name,val->value);
     399          map* length=getMap(input->content,"length");
     400          if(length!=NULL){
     401            char* value=NULL;
     402            int len=atoi(length->value);
     403            int i=0;
     404            for(i=0;i<len;i++){
     405              map* val=getMapArray(input->content,"value",i);
     406              if(val!=NULL){
     407                if(value==NULL){
     408                  value=(char*)malloc((strlen(val->value)+3)*sizeof(char));
     409                  sprintf(value,"\"%s\"",val->value);
     410                }
     411                else{
     412                  value=(char*)realloc(value,(strlen(value)+strlen(val->value)+4)*sizeof(char));
     413                  sprintf(value,"%s \"%s\"",value,val->value);
     414                }
     415              }
     416            }
     417            if(value!=NULL){
     418              parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(value)+3)*sizeof(char));
     419              sprintf(parameters[parameters_cnt-1],"-%s %s",input->name,value);
     420            }
     421          }else{
     422            map* val=getMap(input->content,"value");
     423            parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(val->value)+5)*sizeof(char));
     424            sprintf(parameters[parameters_cnt-1],"-%s \"%s\"",input->name,val->value);
     425          }
    402426        }
    403427      }
     
    888912        dup2 (fileno (stderr), fileno (stdout));
    889913        conf_read(filePath,m);
     914        //dumpMaps(m);
    890915        fflush(stdout);
    891916        dup2 (saved_stdout, fileno (stdout));
     
    910935                free(targetPath);
    911936              }else{
    912                 map* hpcStdErr=getMapFromMaps(*main_conf,"henv","StdErr");
    913                 if(hpcStdErr!=NULL && ssh_fetch(*main_conf,targetPath,hpcStdErr->value,ssh_get_cnt(m))==0){
     937                map* hpcStdErr=getMapFromMaps(*main_conf,"henv","StdErr");
     938                // Added for using sacct in place of scontrol
     939                char *sourcePath=NULL;
     940                if(hpcStdErr!=NULL){
     941                  sourcePath=(char*)malloc((strlen(targetPathMap->value)+strlen(hpcStdErr->value)+2)*sizeof(char));
     942                  sprintf(sourcePath,"%s/%s",targetPathMap->value,hpcStdErr->value);
     943                }
     944                if(hpcStdErr!=NULL && sourcePath!=NULL && ssh_fetch(*main_conf,targetPath,sourcePath,ssh_get_cnt(m))==0){
     945                  free(sourcePath);
    914946                  struct stat f_status;
    915947                  int ts=stat(targetPath, &f_status);
     
    925957                    free(fcontent);
    926958                  }else{
    927                     char *tmpStr=(char*)malloc((strlen(filename)+strlen(_("Unable to fetch the remote file for %s"))+1)*sizeof(char));
    928                     sprintf(tmpStr,_("Unable to fetch the remote file for %s"),filename);
     959                    char *tmpStr=(char*)malloc((strlen(targetPath)+strlen(_("Unable to fetch the remote file for %s"))+1)*sizeof(char));
     960                    sprintf(tmpStr,_("Unable to fetch the remote file for %s"),targetPath);
    929961                    setMapInMaps(*main_conf,"lenv","message",tmpStr);
    930962                    free(tmpStr);
     
    939971                return SERVICE_FAILED;
    940972              }
    941             }       
     973            }
    942974          }else{
    943975            map* generatedFile=getMap(input->content,"generated_file");
     
    9811013              }else{
    9821014                map* hpcStdErr=getMapFromMaps(*main_conf,"henv","StdErr");
    983                 if(hpcStdErr!=NULL && ssh_fetch(*main_conf,targetPath,hpcStdErr->value,ssh_get_cnt(m))==0){
     1015                char *sourcePath=NULL;
     1016                if(hpcStdErr!=NULL){
     1017                  dumpMap(hpcStdErr);
     1018                  sourcePath=(char*)malloc((strlen(targetPathMap->value)+strlen(hpcStdErr->value)+2)*sizeof(char));
     1019                  sprintf(sourcePath,"%s/%s",targetPathMap->value,hpcStdErr->value);
     1020                }
     1021                if(hpcStdErr!=NULL && sourcePath!=NULL && ssh_fetch(*main_conf,targetPath,sourcePath,ssh_get_cnt(m))==0){
     1022                  free(sourcePath);
    9841023                  struct stat f_status;
    9851024                  int ts=stat(targetPath, &f_status);
     
    9951034                    free(fcontent);
    9961035                  }else{
    997                     char *tmpStr=(char*)malloc((strlen(filename)+strlen(_("Unable to fetch the remote file for %s"))+1)*sizeof(char));
    998                     sprintf(tmpStr,_("Unable to fetch the remote file for %s"),filename);
     1036                    char *tmpStr=(char*)malloc((strlen(targetPath)+strlen(_("Unable to fetch the remote file for %s"))+1)*sizeof(char));
     1037                    sprintf(tmpStr,_("Unable to fetch the remote file for %s"),targetPath);
    9991038                    setMapInMaps(*main_conf,"lenv","message",tmpStr);
    10001039                    free(tmpStr);
    10011040                  }
    10021041                }else{
    1003                   char *tmpStr=(char*)malloc((strlen(filename)+strlen(_("Unable to fetch the remote file for %s"))+1)*sizeof(char));
    1004                   sprintf(tmpStr,_("Unable to fetch the remote file for %s"),filename);
     1042                  char *tmpStr=(char*)malloc((strlen(sourcePath)+strlen(_("Unable to fetch the remote file for %s"))+1)*sizeof(char));
     1043                  sprintf(tmpStr,_("Unable to fetch the remote file for %s"),sourcePath);
    10051044                  setMapInMaps(*main_conf,"lenv","message",tmpStr);
    10061045                  free(tmpStr);
  • branches/prototype-v0/zoo-project/zoo-kernel/service_internal_ms.c

    r880 r886  
    542542  int imyIndex=getPublishedId(output);
    543543  msMapSetExtent(m,minX,minY,maxX,maxY);
    544   m->maxsize=4096;
     544  //m->maxsize=4096;
    545545#ifdef DEBUGMS
    546546  fprintf(stderr,"Extent %.15f %.15f %.15f %.15f\n",minX,minY,maxX,maxY);
     
    951951   */
    952952  GDALAllRegister();
    953   hDataset = GDALOpen( pszFilename, GA_Update );
     953  hDataset = GDALOpen( pszFilename, GA_Update ); /*GA_ReadOnly*/
    954954  if( hDataset == NULL ){
    955955#ifdef DEBUGMS
     
    10391039    setSrsInformations(output,m,myLayer,pszProjection);
    10401040  }else{
    1041     fprintf(stderr,"NO SRS FOUND ! %s\n",GDALGetProjectionRef( hDataset ));
     1041    fprintf(stderr,"NO SRS FOUND %s %d ! %s\n",__FILE__,__LINE__,GDALGetProjectionRef( hDataset ));
     1042    fflush(stderr);
    10421043    CPLErr sp=GDALSetProjection( hDataset , "+init=epsg:4326" );
    10431044    if(sp!=CE_None){
    1044       fprintf(stderr,"NO SRS FOUND ! %s\n",CPLGetLastErrorMsg());
     1045      fprintf(stderr,"NO SRS SET ! %s\n",CPLGetLastErrorMsg());
    10451046    }
    10461047  }
     
    11331134
    11341135    CPLErrorReset();
    1135     GDALComputeRasterStatistics( hBand, TRUE, &pdfMin, &pdfMax, &pdfMean, &pdfStdDev, NULL,NULL);
     1136    GDALGetRasterStatistics( hBand, TRUE, TRUE, &pdfMin, &pdfMax, &pdfMean, &pdfStdDev);
     1137    //GDALComputeRasterStatistics( hBand, TRUE, &pdfMin, &pdfMax, &pdfMean, &pdfStdDev, NULL,NULL);
    11361138    char tmpN[21];
    11371139    sprintf(tmpN,"Band%d",iBand+1);
     
    11431145      sprintf(tmpI,"%s_interval",tmpN);
    11441146      msInsertHashTable(&(myLayer->metadata), tmpI, tmpMm);
    1145       if(pdfMax>255)
    1146         msLayerAddProcessing(myLayer,tmpMp);
     1147      //if(pdfMax>255)
     1148      msLayerAddProcessing(myLayer,tmpMp);
    11471149      map* test=getMap(output->content,"msClassify");
    11481150      if(test!=NULL && strncasecmp(test->value,"true",4)==0){
     
    14981500    fprintf(myMaps,"%s\n",mapfile);
    14991501    fclose(myMaps);
    1500   }
     1502  }
     1503  free(mapName);
     1504  free(tmp);
    15011505}
  • branches/prototype-v0/zoo-project/zoo-kernel/sshapi.c

    r877 r886  
    438438    }
    439439    if(!sessions[cnt]->sftp_session)
    440       zSleep(10);
     440      zSleep(1);
    441441  } while (!sessions[cnt]->sftp_session);
    442442  do {
     
    453453    }
    454454    if(!sftp_handle)
    455       zSleep(100);
     455      zSleep(1);
    456456  } while (!sftp_handle);
    457457 
     
    530530  char *logPath=(char*)malloc((strlen(tmpPath->value)+strlen(uuid->value)+11)*sizeof(char));
    531531  sprintf(logPath,"%s/exec_out_%s",tmpPath->value,uuid->value);
     532 
    532533  FILE* logFile=fopen(logPath,"wb");
    533534  free(logPath);
     
    542543        bytecount += rc;
    543544        buffer[rc]=0;
     545       
    544546        fprintf(logFile,"%s",buffer);
    545547        fflush(logFile);
     
    649651}
    650652
     653int fileMd5Check(maps** conf,const char* localPath,const char* targetPath){
     654  if(strstr(localPath,".zca")!=NULL){
     655    char *logPath=NULL;
     656    char *command=(char*)malloc((strlen(targetPath)+27)*sizeof(char));
     657    sprintf(command,"md5sum %s | awk {'print $1'}",targetPath);
     658    if(ssh_exec(*conf,command,ssh_get_cnt(*conf))<=0){
     659      return -1;
     660    }else{
     661      struct stat f_status={};
     662      map* usid=getMapFromMaps(*conf,"lenv","usid");
     663      map* tmpMap=getMapFromMaps(*conf,"main","tmpPath");
     664      char* tmpPath=zStrdup(localPath);
     665      tmpPath[strlen(tmpPath)-2]='m';
     666      tmpPath[strlen(tmpPath)-1]='d';
     667      free(command);
     668      logPath=(char*)malloc((strlen(tmpMap->value)+strlen(usid->value)+11)*sizeof(char));
     669      sprintf(logPath,"%s/exec_out_%s",tmpMap->value,usid->value);
     670      int ts=stat(logPath, &f_status);
     671      if(ts==0) {
     672        char* fcontent=(char*)malloc(sizeof(char)*(f_status.st_size+1));
     673        FILE* f=fopen(logPath,"rb");
     674        fread(fcontent,f_status.st_size,1,f);
     675        fcontent[f_status.st_size-1]=0;
     676        fclose(f);
     677        free(logPath);
     678        struct stat f_status1={};
     679        int ts1=stat(tmpPath, &f_status1);
     680        if(ts1==0) {
     681          char* fcontent1=(char*)malloc(sizeof(char)*(f_status.st_size+1));
     682          FILE* f1=fopen(tmpPath,"rb");
     683          fread(fcontent1,f_status1.st_size,1,f1);
     684          fcontent1[f_status1.st_size]=0;
     685          fclose(f1);
     686          free(tmpPath);
     687          if(strcmp(fcontent,fcontent1)==0){
     688            free(fcontent);
     689            free(fcontent1);
     690            return 0;
     691          }else{
     692            free(fcontent);
     693            free(fcontent1);
     694            return -1;
     695          }
     696        }else{
     697          free(tmpPath);
     698          free(fcontent);
     699          return -1;
     700        }       
     701      }
     702      free(logPath);
     703      free(tmpPath);
     704    }
     705  }
     706  return -1;
     707}
     708
    651709bool runUpload(maps** conf){
    652710  SSHCON *test=ssh_connect(*conf);
     
    665723        getMapArray(queueMaps->content,"targetPath",i)
    666724      };
    667       /**/zooLock* lck;
    668       if((lck=lockFile(*conf,argv[1]->value,'w'))!=NULL){/**/
    669         if(ssh_copy(*conf,argv[1]->value,argv[2]->value,ssh_get_cnt(*conf))!=true){
    670           char* templateStr=_("Unable to copy over SSH the file requested for setting the value of %s.");
    671           char *tmpMessage=(char*)malloc((strlen(templateStr)+strlen(argv[0]->value)+1)*sizeof(char));
    672           sprintf(tmpMessage,templateStr,argv[0]->value);
    673           setMapInMaps(*conf,"lenv","message",tmpMessage);
    674           free(tmpMessage);
    675           unlockFile(*conf,lck);
     725      if(fileMd5Check(conf,argv[1]->value,argv[2]->value)<0){
     726        /**/zooLock* lck;
     727        if((lck=lockFile(*conf,argv[1]->value,'w'))!=NULL){/**/
     728          if(ssh_copy(*conf,argv[1]->value,argv[2]->value,ssh_get_cnt(*conf))!=true){
     729            char* templateStr=_("Unable to copy over SSH the file requested for setting the value of %s.");
     730            char *tmpMessage=(char*)malloc((strlen(templateStr)+strlen(argv[0]->value)+1)*sizeof(char));
     731            sprintf(tmpMessage,templateStr,argv[0]->value);
     732            setMapInMaps(*conf,"lenv","message",tmpMessage);
     733            free(tmpMessage);
     734            unlockFile(*conf,lck);
     735            return false;
     736          }
     737          /**/unlockFile(*conf,lck);
     738        }else{
     739          setMapInMaps(*conf,"lenv","message",_("Unable to lock the file for upload!"));
    676740          return false;
    677         }
    678         /**/unlockFile(*conf,lck);
    679       }else{
    680         setMapInMaps(*conf,"lenv","message",_("Unable to lock the file for upload!"));
    681         return false;
    682       }/**/
     741        }/**/
     742      }
    683743    }   
    684744  }
Note: See TracChangeset for help on using the changeset viewer.

Search

Context Navigation

ZOO Sponsors

http://www.zoo-project.org/trac/chrome/site/img/geolabs-logo.pnghttp://www.zoo-project.org/trac/chrome/site/img/neogeo-logo.png http://www.zoo-project.org/trac/chrome/site/img/apptech-logo.png http://www.zoo-project.org/trac/chrome/site/img/3liz-logo.png http://www.zoo-project.org/trac/chrome/site/img/gateway-logo.png

Become a sponsor !

Knowledge partners

http://www.zoo-project.org/trac/chrome/site/img/ocu-logo.png http://www.zoo-project.org/trac/chrome/site/img/gucas-logo.png http://www.zoo-project.org/trac/chrome/site/img/polimi-logo.png http://www.zoo-project.org/trac/chrome/site/img/fem-logo.png http://www.zoo-project.org/trac/chrome/site/img/supsi-logo.png http://www.zoo-project.org/trac/chrome/site/img/cumtb-logo.png

Become a knowledge partner

Related links

http://zoo-project.org/img/ogclogo.png http://zoo-project.org/img/osgeologo.png