Index: branches/prototype-v0/zoo-project/zoo-kernel/caching.c =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/caching.c (revision 885) +++ branches/prototype-v0/zoo-project/zoo-kernel/caching.c (revision 886) @@ -177,4 +177,5 @@ fwrite(md5fstr,sizeof(char),strlen(md5fstr),fo); free(md5fstr); + free(storage); fclose(fo); return 0; Index: branches/prototype-v0/zoo-project/zoo-kernel/meta_sql.c =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/meta_sql.c (revision 885) +++ branches/prototype-v0/zoo-project/zoo-kernel/meta_sql.c (revision 886) @@ -42,7 +42,7 @@ #endif #define META_SERVICES_LIST_ALL \ - "select id,identifier,title,abstract,service_type,service_provider"\ + "select id,identifier,title,abstract,service_type,service_provider,conf_id"\ " from ows_process" -#define META_SERVICES_LIST_ALL_LENGTH 83 +#define META_SERVICES_LIST_ALL_LENGTH strlen(META_SERVICES_LIST_ALL) #define META_SERVICES_KEYWORDS_FROM_PROCESS \ @@ -50,5 +50,5 @@ " (SELECT keywords_id FROM CollectionDB.DescriptionsKeywordsAssignment"\ " where descriptions_id=%s) " -#define META_SERVICES_KEYWORDS_FROM_PROCESS_LENGTH 162 +#define META_SERVICES_KEYWORDS_FROM_PROCESS_LENGTH strlen(META_SERVICES_KEYWORDS_FROM_PROCESS) #define META_SERVICES_META_FROM_ANYTHING \ @@ -56,5 +56,5 @@ " (SELECT metadata_id FROM CollectionDB.DescriptionsMetadataAssignment"\ " where descriptions_id=%s) " -#define META_SERVICES_META_FROM_ANYTHING_LENGTH 162 +#define META_SERVICES_META_FROM_ANYTHING_LENGTH strlen(META_SERVICES_META_FROM_ANYTHING) #define META_SERVICES_AP_FROM_ANYTHING \ @@ -62,33 +62,33 @@ " (SELECT additional_parameters_id FROM CollectionDB.DescriptionsAdditionalParametersAssignment"\ " where descriptions_id=%s) " -#define META_SERVICES_AP_FROM_ANYTHING_LENGTH 199 +#define META_SERVICES_AP_FROM_ANYTHING_LENGTH strlen(META_SERVICES_AP_FROM_ANYTHING) #define META_SERVICES_AP_FROM_AP \ "SELECT key,value FROM CollectionDB.ows_AdditionalParameter where additional_parameters_id =$q$%s$q$" -#define META_SERVICES_AP_FROM_AP_LENGTH 100 +#define META_SERVICES_AP_FROM_AP_LENGTH strlen(META_SERVICES_AP_FROM_AP) #define META_SERVICES_LIST_INPUTS_FROM_PROCESS \ "select id, identifier,title,abstract,min_occurs,max_occurs from CollectionDB.ows_Input where id in (SELECT input_id from CollectionDB.ProcessInputAssignment where process_id=%s) order by id" -#define META_SERVICES_LIST_INPUTS_FROM_PROCESS_LENGTH 190 +#define META_SERVICES_LIST_INPUTS_FROM_PROCESS_LENGTH strlen(META_SERVICES_LIST_INPUTS_FROM_PROCESS) #define META_SERVICES_LIST_INPUTS_FROM_INPUT \ "select id, identifier,title,abstract,min_occurs,max_occurs from CollectionDB.ows_Input where id in (SELECT child_input from CollectionDB.InputInputAssignment where parent_input=%s) order by id" -#define META_SERVICES_LIST_INPUTS_FROM_INPUT_LENGTH 193 +#define META_SERVICES_LIST_INPUTS_FROM_INPUT_LENGTH strlen(META_SERVICES_LIST_INPUTS_FROM_INPUT) #define META_SERVICES_LIST_OUTPUTS_FROM_PROCESS \ "select id, identifier,title,abstract from CollectionDB.ows_Output where id in (SELECT output_id from CollectionDB.ProcessOutputAssignment where process_id=%s) order by id" -#define META_SERVICES_LIST_OUTPUTS_FROM_PROCESS_LENGTH 171 +#define META_SERVICES_LIST_OUTPUTS_FROM_PROCESS_LENGTH strlen(META_SERVICES_LIST_OUTPUTS_FROM_PROCESS) #define META_SERVICES_LIST_OUTPUTS_FROM_OUTPUT \ "select id, identifier,title,abstract from CollectionDB.ows_Output where id in (SELECT child_output from CollectionDB.OutputOutputAssignment where parent_output=%s) order by id" -#define META_SERVICES_LIST_OUTPUTS_FROM_OUTPUT_LENGTH 176 +#define META_SERVICES_LIST_OUTPUTS_FROM_OUTPUT_LENGTH strlen(META_SERVICES_LIST_OUTPUTS_FROM_OUTPUT) #define META_SERVICES_LIST_LITERAL_FROM_IO \ "select (SELECT name as type FROM CollectionDB.PrimitiveDatatypes where CollectionDB.PrimitiveDatatypes.id=data_type_id),default_value,(SELECT uom from CollectionDB.PrimitiveUOM where id=CollectionDB.LiteralDataDomain.uom),translate(translate(ARRAY((SELECT allowed_Value from CollectionDB.AllowedValues where id in (SELECT allowed_value_id from CollectionDB.AllowedValuesAssignment where literal_data_domain_id=CollectionDB.LiteralDataDomain.id)))::varchar,'{',''),'}',''),def as allowedvalues from CollectionDB.LiteralDataDomain where id in (SELECT data_description_id from CollectionDB.%sDataDescriptionAssignment where %s_id = %s);" -#define META_SERVICES_LIST_LITERAL_FROM_IO_LENGTH 634 +#define META_SERVICES_LIST_LITERAL_FROM_IO_LENGTH strlen(META_SERVICES_LIST_LITERAL_FROM_IO) #define META_SERVICES_LIST_FORMATS_FROM_IO \ "select mime_type,encoding,schema,maximum_megabytes,CASE WHEN use_mapserver THEN 'true' ELSE 'false' END, ms_styles, def from CollectionDB.ows_Format,CollectionDB.PrimitiveFormats where CollectionDB.ows_Format.primitive_format_id=CollectionDB.PrimitiveFormats.id and CollectionDB.ows_Format.id in (SELECT format_id from collectiondb.ows_datadescription where id in ( SELECT data_description_id from CollectionDB.%sDataDescriptionAssignment where %s_id = %s))" -#define META_SERVICES_LIST_FORMATS_FROM_IO_LENGTH 458 +#define META_SERVICES_LIST_FORMATS_FROM_IO_LENGTH strlen(META_SERVICES_LIST_FORMATS_FROM_IO) /** @@ -387,4 +387,5 @@ addToMap(s->content,"serviceType",poFeature->GetFieldAsString( 4 )); addToMap(s->content,"serviceProvider",poFeature->GetFieldAsString( 5 )); + addToMap(s->content,"confId",poFeature->GetFieldAsString( 6 )); addToMap(s->content,"fromDb","true"); s->metadata=NULL; Index: branches/prototype-v0/zoo-project/zoo-kernel/service_callback.c =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/service_callback.c (revision 885) +++ branches/prototype-v0/zoo-project/zoo-kernel/service_callback.c (revision 886) @@ -769,5 +769,6 @@ char *keys[6][2]={ { - "SubmitTime", + //"SubmitTime", + "Submit", "hpc_submission_date" }, @@ -781,13 +782,16 @@ }, { - "StartTime", + //"StartTime", + "Start", "hpc_start_date" }, { - "EndTime", + //"EndTime", + "End", "hpc_end_date" }, { - "JobState", + //"JobState", + "State", "hpc_status" } Index: branches/prototype-v0/zoo-project/zoo-kernel/service_internal_hpc.c =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/service_internal_hpc.c (revision 885) +++ branches/prototype-v0/zoo-project/zoo-kernel/service_internal_hpc.c (revision 886) @@ -299,5 +299,5 @@ // then, set the configId key in the lenv section char *serviceType; - map* mServiceType=getMap(s->content,"serviceType"); + map* mServiceType=getMap(s->content,"confId"); if(mServiceType!=NULL) serviceType=mServiceType->value; @@ -334,5 +334,5 @@ while(input!=NULL && input->content!=NULL){ map* isInRequest=getMap(input->content,"inRequest"); - map* minNb=getMap(input->content,"minoccurs"); + map* minNb=getMap(input->content,"minOccurs"); if(getMaps(*real_outputs,input->name)==NULL && ( (isInRequest!=NULL && strncasecmp(isInRequest->value,"true",4)==0) @@ -397,7 +397,31 @@ if(getMap(input->content,"dataType")!=NULL){ // For LitteralData, simply pass the value - map* val=getMap(input->content,"value"); - parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(val->value)+3)*sizeof(char)); - sprintf(parameters[parameters_cnt-1],"-%s %s",input->name,val->value); + map* length=getMap(input->content,"length"); + if(length!=NULL){ + char* value=NULL; + int len=atoi(length->value); + int i=0; + for(i=0;icontent,"value",i); + if(val!=NULL){ + if(value==NULL){ + value=(char*)malloc((strlen(val->value)+3)*sizeof(char)); + sprintf(value,"\"%s\"",val->value); + } + else{ + value=(char*)realloc(value,(strlen(value)+strlen(val->value)+4)*sizeof(char)); + sprintf(value,"%s \"%s\"",value,val->value); + } + } + } + if(value!=NULL){ + parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(value)+3)*sizeof(char)); + sprintf(parameters[parameters_cnt-1],"-%s %s",input->name,value); + } + }else{ + map* val=getMap(input->content,"value"); + parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(val->value)+5)*sizeof(char)); + sprintf(parameters[parameters_cnt-1],"-%s \"%s\"",input->name,val->value); + } } } @@ -888,4 +912,5 @@ dup2 (fileno (stderr), fileno (stdout)); conf_read(filePath,m); + //dumpMaps(m); fflush(stdout); dup2 (saved_stdout, fileno (stdout)); @@ -910,6 +935,13 @@ free(targetPath); }else{ - map* hpcStdErr=getMapFromMaps(*main_conf,"henv","StdErr"); - if(hpcStdErr!=NULL && ssh_fetch(*main_conf,targetPath,hpcStdErr->value,ssh_get_cnt(m))==0){ + map* hpcStdErr=getMapFromMaps(*main_conf,"henv","StdErr"); + // Added for using sacct in place of scontrol + char *sourcePath=NULL; + if(hpcStdErr!=NULL){ + sourcePath=(char*)malloc((strlen(targetPathMap->value)+strlen(hpcStdErr->value)+2)*sizeof(char)); + sprintf(sourcePath,"%s/%s",targetPathMap->value,hpcStdErr->value); + } + if(hpcStdErr!=NULL && sourcePath!=NULL && ssh_fetch(*main_conf,targetPath,sourcePath,ssh_get_cnt(m))==0){ + free(sourcePath); struct stat f_status; int ts=stat(targetPath, &f_status); @@ -925,6 +957,6 @@ free(fcontent); }else{ - char *tmpStr=(char*)malloc((strlen(filename)+strlen(_("Unable to fetch the remote file for %s"))+1)*sizeof(char)); - sprintf(tmpStr,_("Unable to fetch the remote file for %s"),filename); + char *tmpStr=(char*)malloc((strlen(targetPath)+strlen(_("Unable to fetch the remote file for %s"))+1)*sizeof(char)); + sprintf(tmpStr,_("Unable to fetch the remote file for %s"),targetPath); setMapInMaps(*main_conf,"lenv","message",tmpStr); free(tmpStr); @@ -939,5 +971,5 @@ return SERVICE_FAILED; } - } + } }else{ map* generatedFile=getMap(input->content,"generated_file"); @@ -981,5 +1013,12 @@ }else{ map* hpcStdErr=getMapFromMaps(*main_conf,"henv","StdErr"); - if(hpcStdErr!=NULL && ssh_fetch(*main_conf,targetPath,hpcStdErr->value,ssh_get_cnt(m))==0){ + char *sourcePath=NULL; + if(hpcStdErr!=NULL){ + dumpMap(hpcStdErr); + sourcePath=(char*)malloc((strlen(targetPathMap->value)+strlen(hpcStdErr->value)+2)*sizeof(char)); + sprintf(sourcePath,"%s/%s",targetPathMap->value,hpcStdErr->value); + } + if(hpcStdErr!=NULL && sourcePath!=NULL && ssh_fetch(*main_conf,targetPath,sourcePath,ssh_get_cnt(m))==0){ + free(sourcePath); struct stat f_status; int ts=stat(targetPath, &f_status); @@ -995,12 +1034,12 @@ free(fcontent); }else{ - char *tmpStr=(char*)malloc((strlen(filename)+strlen(_("Unable to fetch the remote file for %s"))+1)*sizeof(char)); - sprintf(tmpStr,_("Unable to fetch the remote file for %s"),filename); + char *tmpStr=(char*)malloc((strlen(targetPath)+strlen(_("Unable to fetch the remote file for %s"))+1)*sizeof(char)); + sprintf(tmpStr,_("Unable to fetch the remote file for %s"),targetPath); setMapInMaps(*main_conf,"lenv","message",tmpStr); free(tmpStr); } }else{ - char *tmpStr=(char*)malloc((strlen(filename)+strlen(_("Unable to fetch the remote file for %s"))+1)*sizeof(char)); - sprintf(tmpStr,_("Unable to fetch the remote file for %s"),filename); + char *tmpStr=(char*)malloc((strlen(sourcePath)+strlen(_("Unable to fetch the remote file for %s"))+1)*sizeof(char)); + sprintf(tmpStr,_("Unable to fetch the remote file for %s"),sourcePath); setMapInMaps(*main_conf,"lenv","message",tmpStr); free(tmpStr); Index: branches/prototype-v0/zoo-project/zoo-kernel/service_internal_ms.c =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/service_internal_ms.c (revision 885) +++ branches/prototype-v0/zoo-project/zoo-kernel/service_internal_ms.c (revision 886) @@ -542,5 +542,5 @@ int imyIndex=getPublishedId(output); msMapSetExtent(m,minX,minY,maxX,maxY); - m->maxsize=4096; + //m->maxsize=4096; #ifdef DEBUGMS fprintf(stderr,"Extent %.15f %.15f %.15f %.15f\n",minX,minY,maxX,maxY); @@ -951,5 +951,5 @@ */ GDALAllRegister(); - hDataset = GDALOpen( pszFilename, GA_Update ); + hDataset = GDALOpen( pszFilename, GA_Update ); /*GA_ReadOnly*/ if( hDataset == NULL ){ #ifdef DEBUGMS @@ -1039,8 +1039,9 @@ setSrsInformations(output,m,myLayer,pszProjection); }else{ - fprintf(stderr,"NO SRS FOUND ! %s\n",GDALGetProjectionRef( hDataset )); + fprintf(stderr,"NO SRS FOUND %s %d ! %s\n",__FILE__,__LINE__,GDALGetProjectionRef( hDataset )); + fflush(stderr); CPLErr sp=GDALSetProjection( hDataset , "+init=epsg:4326" ); if(sp!=CE_None){ - fprintf(stderr,"NO SRS FOUND ! %s\n",CPLGetLastErrorMsg()); + fprintf(stderr,"NO SRS SET ! %s\n",CPLGetLastErrorMsg()); } } @@ -1133,5 +1134,6 @@ CPLErrorReset(); - GDALComputeRasterStatistics( hBand, TRUE, &pdfMin, &pdfMax, &pdfMean, &pdfStdDev, NULL,NULL); + GDALGetRasterStatistics( hBand, TRUE, TRUE, &pdfMin, &pdfMax, &pdfMean, &pdfStdDev); + //GDALComputeRasterStatistics( hBand, TRUE, &pdfMin, &pdfMax, &pdfMean, &pdfStdDev, NULL,NULL); char tmpN[21]; sprintf(tmpN,"Band%d",iBand+1); @@ -1143,6 +1145,6 @@ sprintf(tmpI,"%s_interval",tmpN); msInsertHashTable(&(myLayer->metadata), tmpI, tmpMm); - if(pdfMax>255) - msLayerAddProcessing(myLayer,tmpMp); + //if(pdfMax>255) + msLayerAddProcessing(myLayer,tmpMp); map* test=getMap(output->content,"msClassify"); if(test!=NULL && strncasecmp(test->value,"true",4)==0){ @@ -1498,4 +1500,6 @@ fprintf(myMaps,"%s\n",mapfile); fclose(myMaps); - } + } + free(mapName); + free(tmp); } Index: branches/prototype-v0/zoo-project/zoo-kernel/sshapi.c =================================================================== --- branches/prototype-v0/zoo-project/zoo-kernel/sshapi.c (revision 885) +++ branches/prototype-v0/zoo-project/zoo-kernel/sshapi.c (revision 886) @@ -438,5 +438,5 @@ } if(!sessions[cnt]->sftp_session) - zSleep(10); + zSleep(1); } while (!sessions[cnt]->sftp_session); do { @@ -453,5 +453,5 @@ } if(!sftp_handle) - zSleep(100); + zSleep(1); } while (!sftp_handle); @@ -530,4 +530,5 @@ char *logPath=(char*)malloc((strlen(tmpPath->value)+strlen(uuid->value)+11)*sizeof(char)); sprintf(logPath,"%s/exec_out_%s",tmpPath->value,uuid->value); + FILE* logFile=fopen(logPath,"wb"); free(logPath); @@ -542,4 +543,5 @@ bytecount += rc; buffer[rc]=0; + fprintf(logFile,"%s",buffer); fflush(logFile); @@ -649,4 +651,60 @@ } +int fileMd5Check(maps** conf,const char* localPath,const char* targetPath){ + if(strstr(localPath,".zca")!=NULL){ + char *logPath=NULL; + char *command=(char*)malloc((strlen(targetPath)+27)*sizeof(char)); + sprintf(command,"md5sum %s | awk {'print $1'}",targetPath); + if(ssh_exec(*conf,command,ssh_get_cnt(*conf))<=0){ + return -1; + }else{ + struct stat f_status={}; + map* usid=getMapFromMaps(*conf,"lenv","usid"); + map* tmpMap=getMapFromMaps(*conf,"main","tmpPath"); + char* tmpPath=zStrdup(localPath); + tmpPath[strlen(tmpPath)-2]='m'; + tmpPath[strlen(tmpPath)-1]='d'; + free(command); + logPath=(char*)malloc((strlen(tmpMap->value)+strlen(usid->value)+11)*sizeof(char)); + sprintf(logPath,"%s/exec_out_%s",tmpMap->value,usid->value); + int ts=stat(logPath, &f_status); + if(ts==0) { + char* fcontent=(char*)malloc(sizeof(char)*(f_status.st_size+1)); + FILE* f=fopen(logPath,"rb"); + fread(fcontent,f_status.st_size,1,f); + fcontent[f_status.st_size-1]=0; + fclose(f); + free(logPath); + struct stat f_status1={}; + int ts1=stat(tmpPath, &f_status1); + if(ts1==0) { + char* fcontent1=(char*)malloc(sizeof(char)*(f_status.st_size+1)); + FILE* f1=fopen(tmpPath,"rb"); + fread(fcontent1,f_status1.st_size,1,f1); + fcontent1[f_status1.st_size]=0; + fclose(f1); + free(tmpPath); + if(strcmp(fcontent,fcontent1)==0){ + free(fcontent); + free(fcontent1); + return 0; + }else{ + free(fcontent); + free(fcontent1); + return -1; + } + }else{ + free(tmpPath); + free(fcontent); + return -1; + } + } + free(logPath); + free(tmpPath); + } + } + return -1; +} + bool runUpload(maps** conf){ SSHCON *test=ssh_connect(*conf); @@ -665,20 +723,22 @@ getMapArray(queueMaps->content,"targetPath",i) }; - /**/zooLock* lck; - if((lck=lockFile(*conf,argv[1]->value,'w'))!=NULL){/**/ - if(ssh_copy(*conf,argv[1]->value,argv[2]->value,ssh_get_cnt(*conf))!=true){ - char* templateStr=_("Unable to copy over SSH the file requested for setting the value of %s."); - char *tmpMessage=(char*)malloc((strlen(templateStr)+strlen(argv[0]->value)+1)*sizeof(char)); - sprintf(tmpMessage,templateStr,argv[0]->value); - setMapInMaps(*conf,"lenv","message",tmpMessage); - free(tmpMessage); - unlockFile(*conf,lck); + if(fileMd5Check(conf,argv[1]->value,argv[2]->value)<0){ + /**/zooLock* lck; + if((lck=lockFile(*conf,argv[1]->value,'w'))!=NULL){/**/ + if(ssh_copy(*conf,argv[1]->value,argv[2]->value,ssh_get_cnt(*conf))!=true){ + char* templateStr=_("Unable to copy over SSH the file requested for setting the value of %s."); + char *tmpMessage=(char*)malloc((strlen(templateStr)+strlen(argv[0]->value)+1)*sizeof(char)); + sprintf(tmpMessage,templateStr,argv[0]->value); + setMapInMaps(*conf,"lenv","message",tmpMessage); + free(tmpMessage); + unlockFile(*conf,lck); + return false; + } + /**/unlockFile(*conf,lck); + }else{ + setMapInMaps(*conf,"lenv","message",_("Unable to lock the file for upload!")); return false; - } - /**/unlockFile(*conf,lck); - }else{ - setMapInMaps(*conf,"lenv","message",_("Unable to lock the file for upload!")); - return false; - }/**/ + }/**/ + } } }