-
Notifications
You must be signed in to change notification settings - Fork 346
/
batch.c
1101 lines (1003 loc) · 55.8 KB
/
batch.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*--------------------------------------------------------------------
*
* Copyright (c) 1991-2024 by the GMT Team (https://www.generic-mapping-tools.org/team.html)
* See LICENSE.TXT file for copying and redistribution conditions.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation; version 3 or any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* Contact info: www.generic-mapping-tools.org
*--------------------------------------------------------------------*/
/*
* Author: Paul Wessel
* Date: 12-May-2020
* Version: 6 API
*
* Brief synopsis: gmt batch automates batch processing
*
* batch automates the main processing loop and much of the machinery needed
* to script a batch job sequence. It allows for optional preflight and
* and postflight scripts to be specified via separate modern mode scripts and
* a single processing script that uses special variables to use slightly
* different parameters for each job. The user only needs to compose these
* simple one-job scripts and then batch takes care of the automation.
* Jobs are run in parallel without need for OpenMP, etc.
*/
#include "gmt_dev.h"
#include "longopt/batch_inc.h"
#ifdef WIN32
#include <windows.h>
#endif
#define THIS_MODULE_CLASSIC_NAME "batch"
#define THIS_MODULE_MODERN_NAME "batch"
#define THIS_MODULE_LIB "core"
#define THIS_MODULE_PURPOSE "Automate batch job processing"
#define THIS_MODULE_KEYS "<D("
#define THIS_MODULE_NEEDS ""
#define THIS_MODULE_OPTIONS "-Vf"
#define BATCH_PREFLIGHT 0
#define BATCH_POSTFLIGHT 1
#define BATCH_WAIT_TO_CHECK 10000 /* In microseconds, so 0.01 seconds */
#define BATCH_PAUSE_A_SEC 1000000 /* In microseconds, so 1 seconds */
/* Control structure for batch */
struct BATCH_CTRL {
struct BATCH_In { /* mainscript (Bourne, Bourne Again, csh, or DOS (bat) script) */
bool active;
enum GMT_enum_script mode;
char *file; /* Name of main script */
FILE *fp; /* Open file pointer to main script */
} In;
struct BATCH_D { /* -D */
bool active;
} D;
struct BATCH_F { /* -F<template> */
bool active;
char *template; /* Name template of product files via -T data */
} F;
struct BATCH_I { /* -I<includefile> */
bool active;
char *file; /* Name of include script */
FILE *fp; /* Open file pointer to include script */
} I;
struct BATCH_M { /* -M[<job>] */
bool active;
bool exit;
unsigned int job; /* Job selected as master job */
} M;
struct BATCH_N { /* -N<batchprefix> */
bool active;
char *prefix; /* Job prefix and also name of working directory (but see -W) */
} N;
struct BATCH_Q { /* -Q[s] */
bool active;
bool scripts;
} Q;
struct BATCH_S { /* -Sb|f<script> */
bool active;
char *file; /* Name of script file */
FILE *fp; /* Open file pointer to script */
} S[2];
struct BATCH_T { /* -T<n_jobs>|<min>/<max/<inc>[+n]|<timefile>[+p<precision>][+s<job>][+w[<str>]] */
bool active;
bool split; /* true means we must split any trailing text in to words, using separators in <str> [" \t"] */
unsigned int n_jobs; /* Total number of jobs */
unsigned int start_job; /* First job [0] */
unsigned int precision; /* Decimals used in making unique job tags */
char sep[GMT_LEN8]; /* word separator(s) */
char *file; /* timefile name */
} T;
struct BATCH_W { /* -W<workingdirectory> */
bool active;
char *dir; /* Alternative working directory than implied by -N */
} W;
struct BATCH_Z { /* -Z[ */
bool active; //* Delete all files including the mainscript and anything passed via -I, -S */
} Z;
struct BATCH_x { /* -x[[-]<ncores>] */
bool active;
int n_threads;
} x;
};
struct BATCH_STATUS {
/* Used to monitor the start, running, and completion of job jobs running in parallel */
bool started; /* true if job job has started */
bool completed; /* true if the completion file has been successfully produced */
};
static void *New_Ctrl (struct GMT_CTRL *GMT) { /* Allocate and initialize a new control structure */
struct BATCH_CTRL *C;
C = gmt_M_memory (GMT, NULL, 1, struct BATCH_CTRL);
C->x.n_threads = GMT->parent->n_cores; /* Use all cores available unless -x is set */
strcpy (C->T.sep, "\t "); /* Any white space */
return (C);
}
static void Free_Ctrl (struct GMT_CTRL *GMT, struct BATCH_CTRL *C) { /* Deallocate control structure */
gmt_M_unused (GMT);
if (!C) return;
gmt_M_str_free (C->In.file);
gmt_M_str_free (C->F.template);
gmt_M_str_free (C->I.file);
gmt_M_str_free (C->N.prefix);
gmt_M_str_free (C->S[BATCH_PREFLIGHT].file);
gmt_M_str_free (C->S[BATCH_POSTFLIGHT].file);
gmt_M_str_free (C->T.file);
gmt_M_str_free (C->W.dir);
gmt_M_free (GMT, C);
}
/*! -x[[-]<ncores>] parsing needed but here not related to OpenMP etc - it is just a local option */
GMT_LOCAL int batch_parse_x_option (struct GMT_CTRL *GMT, struct BATCH_CTRL *Ctrl, char *arg) {
if (!arg) return (GMT_PARSE_ERROR); /* -x requires a non-NULL argument */
if (arg[0])
Ctrl->x.n_threads = atoi (arg);
if (Ctrl->x.n_threads == 0) /* Not having any of that. At least one */
Ctrl->x.n_threads = 1;
else if (Ctrl->x.n_threads < 0) /* Meant to reduce the number of threads */
Ctrl->x.n_threads = MAX(GMT->parent->n_cores + Ctrl->x.n_threads, 1); /* Max-n but at least one */
return (GMT_NOERROR);
}
GMT_LOCAL void batch_close_files (struct BATCH_CTRL *Ctrl) {
/* Close all files when an error forces us to quit */
fclose (Ctrl->In.fp);
if (Ctrl->I.active) fclose (Ctrl->I.fp);
}
GMT_LOCAL int batch_delete_scripts (struct GMT_CTRL *GMT, struct BATCH_CTRL *Ctrl) {
/* Delete the scripts since they apparently are temporary */
if (Ctrl->In.file && gmt_remove_file (GMT, Ctrl->In.file)) { /* Delete the main script */
GMT_Report (GMT->parent, GMT_MSG_ERROR, "Unable to delete the main script %s.\n", Ctrl->In.file);
return (GMT_RUNTIME_ERROR);
}
if (Ctrl->I.file && gmt_remove_file (GMT, Ctrl->I.file)) { /* Delete the include script */
GMT_Report (GMT->parent, GMT_MSG_ERROR, "Unable to delete the include script %s.\n", Ctrl->I.file);
return (GMT_RUNTIME_ERROR);
}
if (Ctrl->S[BATCH_PREFLIGHT].file && gmt_remove_file (GMT, Ctrl->S[BATCH_PREFLIGHT].file)) { /* Delete the background script */
GMT_Report (GMT->parent, GMT_MSG_ERROR, "Unable to delete the preflight script %s.\n", Ctrl->S[BATCH_PREFLIGHT].file);
return (GMT_RUNTIME_ERROR);
}
if (Ctrl->S[BATCH_POSTFLIGHT].file && gmt_remove_file (GMT, Ctrl->S[BATCH_POSTFLIGHT].file)) { /* Delete the foreground script */
GMT_Report (GMT->parent, GMT_MSG_ERROR, "Unable to delete the postflight script %s.\n", Ctrl->S[BATCH_POSTFLIGHT].file);
return (GMT_RUNTIME_ERROR);
}
return (GMT_NOERROR);
}
GMT_LOCAL void batch_set_product_prefix (struct GMT_CTRL *GMT, struct GMT_DATASET *D, unsigned int row, unsigned int n_fmts, char *template, char *custom_name) {
/* Create the custom product file prefix */
struct GMT_DATASEGMENT *S = D->table[0]->segment[0]; /* Current (and only) segment */
gmt_M_unused (GMT);
if (D->type == GMT_READ_DATA) { /* No trailing text used */
switch (n_fmts) { /* May use 1-5 numerical columns */
case 1: sprintf (custom_name, template, S->data[0][row]); break;
case 2: sprintf (custom_name, template, S->data[0][row], S->data[1][row]); break;
case 3: sprintf (custom_name, template, S->data[0][row], S->data[1][row], S->data[2][row]); break;
case 4: sprintf (custom_name, template, S->data[0][row], S->data[1][row], S->data[2][row], S->data[3][row]); break;
case 5: sprintf (custom_name, template, S->data[0][row], S->data[1][row], S->data[2][row], S->data[3][row], S->data[4][row]); break;
default: break; /* We have checked it is no more than 5 */
}
}
else { /* Use the trailing text as well */
switch (n_fmts) { /* May use 0-5 numerical columns since trailing text is also used */
case 0: sprintf (custom_name, template, S->text[row]); break;
case 1: sprintf (custom_name, template, S->data[0][row], S->text[row]); break;
case 2: sprintf (custom_name, template, S->data[0][row], S->data[1][row], S->text[row]); break;
case 3: sprintf (custom_name, template, S->data[0][row], S->data[1][row], S->data[2][row], S->text[row]); break;
case 4: sprintf (custom_name, template, S->data[0][row], S->data[1][row], S->data[2][row], S->data[3][row], S->text[row]); break;
case 5: sprintf (custom_name, template, S->data[0][row], S->data[1][row], S->data[2][row], S->data[3][row], S->data[4][row], S->text[row]); break;
default: break; /* We have checked it is no more than 5 */
}
}
gmt_strrepc (custom_name, ' ', '_'); /* Replace any spaces with underscores to avoid template disasters */
gmt_strrepc (custom_name, '\t', '_'); /* Replace any tabs with underscores to avoid template disasters */
}
static int usage (struct GMTAPI_CTRL *API, int level) {
const char *name = gmt_show_name_and_purpose (API, THIS_MODULE_LIB, THIS_MODULE_CLASSIC_NAME, THIS_MODULE_PURPOSE);
if (level == GMT_MODULE_PURPOSE) return (GMT_NOERROR);
GMT_Usage (API, 0, "usage: %s <mainscript> -N<prefix> -T<njobs>|<min>/<max>/<inc>[+n]|<timefile>[+p<width>][+s<first>][+w[<str>]|W] "
"[-D] [-F<template>] -I<includefile>] [-M[<job>]] [-Q[s]] [-Sb<postflight>] [-Sf<preflight>] "
"[%s] [-W[<dir>]] [-Z] [%s] [-x[[-]<n>]] [%s]\n", name, GMT_V_OPT, GMT_f_OPT, GMT_PAR_OPT);
if (level == GMT_SYNOPSIS) return (GMT_MODULE_SYNOPSIS);
GMT_Message (API, GMT_TIME_NONE, " REQUIRED ARGUMENTS:\n");
GMT_Usage (API, 1, "\n<mainscript>");
GMT_Usage (API, -2, "The main GMT modern script that completes a single job.");
GMT_Usage (API, 1, "\n-N<prefix>");
GMT_Usage (API, -2, "Set the <prefix> used for batch files and directory names.");
GMT_Usage (API, 1, "\n-T<njobs>|<min>/<max>/<inc>[+n]|<timefile>[+p<width>][+s<first>][+w[<str>]|W]");
GMT_Usage (API, -2, "Set number of jobs, create parameters from <min>/<max>/<inc>[+n] or give file with job-specific information. "
"If <timefile> does not exist it must be created by the preflight script given via -Sf.");
GMT_Usage (API, 3, "+p Set number of digits used in creating the job tags [automatic].");
GMT_Usage (API, 3, "+n Indicate that <inc> in <min>/<max>/<inc> is in fact number of jobs instead.");
GMT_Usage (API, 3, "+s Append <first> to change the value of the first job [0].");
GMT_Usage (API, 3, "+w Let trailing text in <timefile> be split into individual word variables. "
"We use space or TAB as separators; append <str> to set custom characters as separators instead.");
GMT_Usage (API, 3, "+W Same as +w but only use TAB as separator.");
GMT_Message (API, GMT_TIME_NONE, "\n OPTIONAL ARGUMENTS:\n");
GMT_Usage (API, 1, "\n-D");
GMT_Usage (API, -2, "No product files to move to the main directory [Default assumes there are named products].");
GMT_Usage (API, 1, "\n-F<template>");
GMT_Usage (API, -2, "Create unique BATCH_NAME file prefixes using the <template> and the columns in <timefile> [use running job numbers]. "
"Requires -T with at least the same number of columns as format statements in <template>.");
GMT_Usage (API, 1, "\n-I<includefile>");
GMT_Usage (API, -2, "Specify a script file to be inserted into the batch_init.sh script [none]. "
"Used to add constant variables needed by all batch scripts.");
GMT_Usage (API, 1, "\n-M[<job>]");
GMT_Usage (API, -2, "Run just the indicated job number [0] for testing [run all].");
GMT_Usage (API, 1, "\n-Q[s]");
GMT_Usage (API, -2, "Debugging: Leave all intermediate files and directories behind for inspection. "
"Append s to only create the work scripts but none will be executed (except for <preflight> script).");
GMT_Usage (API, 1, "\n-Sf<preflight>");
GMT_Usage (API, -2, "Append name of <preflight> GMT modern script that may download or compute "
"files needed by the <mainscript>.");
GMT_Usage (API, 1, "\n-Sb<postflight>");
GMT_Usage (API, -2, "Append name of <postflight> script (not necessarily a GMT script) which will "
"take actions once all batch jobs have completed.");
GMT_Option (API, "V");
GMT_Usage (API, 1, "\n-W[<dir>]");
GMT_Usage (API, -2, "Specify <dir> where temporary files will be built [<dir> = <prefix> set by -N]. "
"If <dir> is not given we create one in the system temp directory named <prefix> (from -N).");
GMT_Usage (API, 1, "\n-Z");
GMT_Usage (API, -2, "Erase input scripts (<mainscript> and any files via -I, -S) [leave input scripts alone]. Not compatible with -Q.");
GMT_Option (API, "f");
/* Number of threads (re-purposed from -x in GMT_Option since this local option is always available and we are not using OpenMP) */
GMT_Usage (API, 1, "\n-x[[-]<n>]");
GMT_Usage (API, -2, "Limit the number of cores used in job generation [Default uses all %d cores]. "
"If <n> is negative then we select (%d - <n>) cores (or at least 1). Note: 1 core is used by batch itself.", API->n_cores, API->n_cores);
GMT_Option (API, ".");
return (GMT_MODULE_USAGE);
}
static int parse (struct GMT_CTRL *GMT, struct BATCH_CTRL *Ctrl, struct GMT_OPTION *options) {
/* This parses the options provided to batch and sets parameters in CTRL.
* Any GMT common options will override values set previously by other commands.
*/
unsigned int n_errors = 0, k, pos;
char p[GMT_LEN256] = {""}, *c = NULL;
struct GMT_OPTION *opt = NULL;
struct GMTAPI_CTRL *API = GMT->parent;
for (opt = options; opt; opt = opt->next) {
switch (opt->option) {
case '<': /* Input file */
n_errors += gmt_M_repeated_module_option (API, Ctrl->In.active);
n_errors += gmt_get_required_file (GMT, opt->arg, opt->option, 0, GMT_IS_DATASET, GMT_IN, GMT_FILE_REMOTE, &(Ctrl->In.file));
break;
case 'D': /* Turn off moving products up to top dir */
n_errors += gmt_M_repeated_module_option (API, Ctrl->D.active);
break;
case 'F': /* Product name template */
n_errors += gmt_M_repeated_module_option (API, Ctrl->F.active);
n_errors += gmt_get_required_string (GMT, opt->arg, opt->option, 0, &Ctrl->F.template);
break;
case 'I': /* Include file with settings used by all scripts */
n_errors += gmt_M_repeated_module_option (API, Ctrl->I.active);
n_errors += gmt_get_required_file (GMT, opt->arg, opt->option, 0, GMT_IS_DATASET, GMT_IN, GMT_FILE_REMOTE, &(Ctrl->I.file));
break;
case 'M': /* Create a single job as well as batch (unless -Q is active) */
n_errors += gmt_M_repeated_module_option (API, Ctrl->M.active);
if (opt->arg[0]) /* Gave a job number [0] */
Ctrl->M.job = atoi (opt->arg);
break;
case 'N': /* Batch job prefix and directory name */
n_errors += gmt_M_repeated_module_option (API, Ctrl->N.active);
n_errors += gmt_get_required_string (GMT, opt->arg, opt->option, 0, &Ctrl->N.prefix);
break;
case 'Q': /* Debug - leave temp files and directories behind; Use -Qs to only write scripts */
n_errors += gmt_M_repeated_module_option (API, Ctrl->Q.active);
if (opt->arg[0] == 's') Ctrl->Q.scripts = true;
break;
case 'S': /* postflight and preflight scripts */
if (opt->arg[0] == 'b')
k = BATCH_PREFLIGHT; /* postflight */
else if (opt->arg[0] == 'f')
k = BATCH_POSTFLIGHT; /* preflight */
else { /* Bad directive */
n_errors++;
GMT_Report (API, GMT_MSG_ERROR, "Option -S: Select -Sb or -Sf\n");
break;
}
n_errors += gmt_M_repeated_module_option (API, Ctrl->S[k].active);
/* Got a valid f or b */
if (gmt_get_required_file (GMT, &opt->arg[1], opt->option, 0, GMT_IS_DATASET, GMT_IN, GMT_FILE_LOCAL, &(Ctrl->S[k].file)))
n_errors++;
else if ((Ctrl->S[k].fp = fopen (Ctrl->S[k].file, "r")) == NULL) {
GMT_Report (API, GMT_MSG_ERROR, "Option -S%c: Unable to open file %s\n", opt->arg[0], Ctrl->S[k].file);
n_errors++;
}
break;
case 'T': /* Number of jobs or the name of file with job information (note: file may not exist yet) */
n_errors += gmt_M_repeated_module_option (API, Ctrl->T.active);
if ((c = gmt_first_modifier (GMT, opt->arg, "psw"))) { /* Process any modifiers */
pos = 0; /* Reset to start of new word */
while (gmt_getmodopt (GMT, 'T', c, "psw", &pos, p, &n_errors) && n_errors == 0) {
switch (p[0]) {
case 'p': /* Set a fixed precision in job naming ###### */
n_errors += gmt_get_required_uint (GMT, opt->arg, opt->option, 'p', &Ctrl->T.precision);
break;
case 's': /* Specify start job other than 0 */
n_errors += gmt_get_required_uint (GMT, opt->arg, opt->option, 's', &Ctrl->T.start_job);
break;
case 'w': /* Split trailing text into words using any white space. */
Ctrl->T.split = true;
if (p[1]) { /* Gave an argument, watch out for tabs given as \t */
char *W = gmt_get_strwithtab (&p[1]);
strncpy (Ctrl->T.sep, W, GMT_LEN8-1);
gmt_M_str_free (W);
}
break;
case 'W': /* Split trailing text into words using only TABs. */
Ctrl->T.split = true;
Ctrl->T.sep[1] = '\0'; /* Truncate the space character in the list to only have TAB */
break;
default:
break; /* These are caught in gmt_getmodopt so break is just for Coverity */
}
}
c[0] = '\0';
}
n_errors += gmt_get_required_string (GMT, opt->arg, opt->option, 0, &Ctrl->T.file);
if (c) c[0] = '+'; /* Restore modifiers */
break;
case 'W': /* Work directory where data files may be found. If not given we make one up later */
n_errors += gmt_M_repeated_module_option (API, Ctrl->W.active);
if (opt->arg[0]) Ctrl->W.dir = strdup (opt->arg);
break;
case 'Z': /* Delete input scripts */
n_errors += gmt_M_repeated_module_option (API, Ctrl->Z.active);
break;
case 'x':
n_errors += gmt_M_repeated_module_option (API, Ctrl->x.active);
n_errors += batch_parse_x_option (GMT, Ctrl, opt->arg);
break;
default: /* Report bad options */
n_errors += gmt_default_option_error (GMT, opt);
break;
}
}
n_errors += gmt_M_check_condition (GMT, Ctrl->In.file == NULL, "Must specify a single main script file\n");
n_errors += gmt_M_check_condition (GMT, !Ctrl->N.active || (Ctrl->N.prefix == NULL || strlen (Ctrl->N.prefix) == 0),
"Option -N: Must specify a batch prefix\n");
n_errors += gmt_M_check_condition (GMT, !Ctrl->T.active,
"Option -T: Must specify number of jobs or a time file\n");
n_errors += gmt_M_check_condition (GMT, Ctrl->T.split && Ctrl->T.sep[0] == '\0',
"Option -T: Must specify a string of characters if using +w<str>\n");
n_errors += gmt_M_check_condition (GMT, Ctrl->Q.active && Ctrl->Z.active,
"Option -Z: Not compatible with -Q\n");
n_errors += gmt_M_check_condition (GMT, Ctrl->M.active && Ctrl->M.job < Ctrl->T.start_job,
"Option -M: Cannot specify a job before the first job number set via -T\n");
n_errors += gmt_M_check_condition (GMT, Ctrl->W.active && Ctrl->W.dir && !strcmp (Ctrl->W.dir, "/tmp"),
"Option -W: Cannot delete working directory %s\n", Ctrl->W.dir);
if (n_errors) return (GMT_PARSE_ERROR); /* No point going further */
/* Note: We open script files for reading below since we are changing cwd later and sometimes need to rewind and re-read */
if (Ctrl->In.file) { /* Determine scripting language from file extension and open the main script file */
if (strstr (Ctrl->In.file, ".bash") || strstr (Ctrl->In.file, ".sh")) /* Treat both as bash since sh is subset of bash */
Ctrl->In.mode = GMT_BASH_MODE;
else if (strstr (Ctrl->In.file, ".csh"))
Ctrl->In.mode = GMT_CSH_MODE;
else if (strstr (Ctrl->In.file, ".bat"))
Ctrl->In.mode = GMT_DOS_MODE;
else {
GMT_Report (API, GMT_MSG_ERROR, "Unable to determine script language from the extension of your script %s\n", Ctrl->In.file);
GMT_Report (API, GMT_MSG_ERROR, "Allowable extensions are: *.sh, *.bash, *.csh, and *.bat\n");
n_errors++;
}
/* Armed with script language we check that any back/fore-ground scripts are of the same kind */
for (k = BATCH_PREFLIGHT; !n_errors && k <= BATCH_POSTFLIGHT; k++) {
if (!Ctrl->S[k].active) continue; /* Not provided */
n_errors += gmt_check_language (GMT, Ctrl->In.mode, Ctrl->S[k].file, k, NULL);
}
if (!n_errors && Ctrl->I.active) { /* Must also check the include file, and open it for reading */
n_errors += gmt_check_language (GMT, Ctrl->In.mode, Ctrl->I.file, 3, NULL);
if (n_errors == 0 && ((Ctrl->I.fp = fopen (Ctrl->I.file, "r")) == NULL)) {
GMT_Report (API, GMT_MSG_ERROR, "Unable to open include script file %s\n", Ctrl->I.file);
n_errors++;
}
}
/* Open the main script for reading here */
if (n_errors == 0 && ((Ctrl->In.fp = fopen (Ctrl->In.file, "r")) == NULL)) {
GMT_Report (API, GMT_MSG_ERROR, "Unable to open main script file %s\n", Ctrl->In.file);
n_errors++;
}
if (n_errors == 0 && gmt_script_is_classic (GMT, Ctrl->In.fp)) {
GMT_Report (API, GMT_MSG_ERROR, "Your main script file %s is not in GMT modern mode\n", Ctrl->In.file);
n_errors++;
}
/* Make sure all BATCH_* variables are used with leading token ($, %) */
if (n_errors == 0 && gmt_token_check (GMT, Ctrl->In.fp, "BATCH_", Ctrl->In.mode)) {
n_errors++;
}
}
return (n_errors ? GMT_PARSE_ERROR : GMT_NOERROR);
}
GMT_LOCAL bool batch_is_product (char *file, char *prefix, size_t len, int precision) {
/* A product file leads with the BATCH_PREFIX and not the BATCH_NAME.
* Since BATCH_NAME = BATCH_PREFIX_##### (number of # is precision) we
* may need to check if those # are all digits as part of the checking. */
size_t L;
if (strncmp (file, prefix, len)) return false; /* Not the right prefix */
if ((L = strlen (file)) == len) return false; /* A directory? */
/* Now must determine difference between prefix and name */
if (L <= (len + precision + 1)) return true; /* Can't be a name so found a product we wish to keep */
for (int k = 0; k < precision; k++) {
if (!isdigit (file[len+1+k])) return true; /* Can't be a name so found a product we wish to keep */
}
return false; /* Must be a name file */
}
#define bailout(code) {gmt_M_free_options (mode); return (code);}
#define Return(code) {Free_Ctrl (GMT, Ctrl); gmt_end_module (GMT, GMT_cpy); bailout (code);}
EXTERN_MSC int GMT_batch (void *V_API, int mode, void *args) {
int error = 0, precision;
int (*run_script)(const char *); /* pointer to system function or a dummy */
size_t P_len = 0;
unsigned int n_values = 0, n_jobs = 0, job, i_job, col, k, n_cores_unused, n_to_run, n_fmts = 0;
unsigned int n_jobs_not_started = 0, n_jobs_completed = 0, first_i_job = 0, data_job;
bool done = false, n_written = false, has_text = false, is_classic = false, has_conf = false;
bool got_time_file = false, issue_col0_par = false;
static char *extension[3] = {"sh", "csh", "bat"}, *load[3] = {"source", "source", "call"}, var_token[4] = "$$%";
static char *rmdir[3] = {"rm -rf", "rm -rf", "rd /s /q"}, *export[3] = {"export ", "setenv ", ""};
static char *mvfile[3] = {"mv -f", "mv -f", "move /Y"};
static char *createfile[3] = {"touch", "touch", "copy /b NUL"}, *rmfile[3] = {"rm -f", "rm -f", "del"};
static char *cpconf[3] = {"\tcp -f %s .\n", "\tcp -f %s .\n", "\tcopy %s .\n"};
static char *sys_cmd_nowait[3] = {"bash", "csh", "start /B"}, *sys_cmd_wait[3] = {"bash", "csh", "cmd /C"};
char init_file[PATH_MAX] = {""}, state_tag[GMT_LEN16] = {""}, state_prefix[GMT_LEN64] = {""}, param_file[PATH_MAX] = {""};
char pre_file[PATH_MAX] = {""}, post_file[PATH_MAX] = {""}, main_file[PATH_MAX] = {""}, line[PATH_MAX] = {""}, tmpwpath[PATH_MAX] = {""};
char string[GMT_LEN128] = {""}, cmd[GMT_LEN256] = {""}, cwd[PATH_MAX] = {""}, conf_file[PATH_MAX];
char completion_file[PATH_MAX] = {""}, topdir[PATH_MAX] = {""}, workdir[PATH_MAX] = {""}, datadir[PATH_MAX] = {""};
char **work_files = NULL;
double percent = 0.0;
FILE *fp = NULL;
struct GMT_CTRL *GMT = NULL, *GMT_cpy = NULL;
struct GMT_DATASET *D = NULL;
struct GMT_OPTION *options = NULL;
struct BATCH_STATUS *status = NULL;
struct BATCH_CTRL *Ctrl = NULL;
struct GMTAPI_CTRL *API = gmt_get_api_ptr (V_API); /* Cast from void to GMTAPI_CTRL pointer */
/*----------------------- Standard module initialization and parsing ----------------------*/
if (API == NULL) return (GMT_NOT_A_SESSION);
if (mode == GMT_MODULE_PURPOSE) return (usage (API, GMT_MODULE_PURPOSE)); /* Return the purpose of program */
options = GMT_Create_Options (API, mode, args); if (API->error) return (API->error); /* Set or get option list */
if ((error = gmt_report_usage (API, options, 0, usage)) != GMT_NOERROR) bailout (error); /* Give usage if requested */
/* Parse the command-line arguments */
if ((GMT = gmt_init_module (API, THIS_MODULE_LIB, THIS_MODULE_CLASSIC_NAME, THIS_MODULE_KEYS, THIS_MODULE_NEEDS, module_kw, &options, &GMT_cpy)) == NULL) bailout (API->error); /* Save current state */
if (GMT_Parse_Common (API, THIS_MODULE_OPTIONS, options)) Return (API->error);
Ctrl = New_Ctrl (GMT); /* Allocate and initialize a new control structure */
if ((error = parse (GMT, Ctrl, options)) != 0) Return (error);
/*---------------------------- This is the batch main code ----------------------------*/
if (Ctrl->Q.scripts) { /* No batch will be run, but scripts will be produced */
GMT_Report (API, GMT_MSG_INFORMATION, "Dry-run enabled - Processing scripts will be created and any pre/post-flight scripts will be executed.\n");
if (Ctrl->M.active) GMT_Report (API, GMT_MSG_INFORMATION, "A single script for job %d will be created and executed\n", Ctrl->M.job);
run_script = gmt_dry_run_only; /* This prevents the main job loop from executing the script */
}
else /* Will run scripts and may even need to make a batch */
run_script = system; /* The standard system function will be used */
/* First try to read -T<timefile> in case it is prescribed directly (and before we change directory) */
if (!gmt_access (GMT, Ctrl->T.file, R_OK)) { /* A file by that name exists and is readable */
if ((D = GMT_Read_Data (API, GMT_IS_DATASET, GMT_IS_FILE, GMT_IS_NONE, GMT_READ_NORMAL, NULL, Ctrl->T.file, NULL)) == NULL) {
GMT_Report (API, GMT_MSG_ERROR, "Unable to read time file: %s - exiting\n", Ctrl->T.file);
batch_close_files (Ctrl);
Return (API->error);
}
if (D->n_segments > 1) { /* We insist on a simple file structure with a single segment */
GMT_Report (API, GMT_MSG_ERROR, "Your time file %s has more than one segment - reformat first\n", Ctrl->T.file);
batch_close_files (Ctrl);
Return (API->error);
}
n_jobs = (unsigned int)D->n_records; /* Number of records means number of jobs */
n_values = (unsigned int)D->n_columns; /* The number of per-job parameters we need to place into the per-job parameter files */
has_text = (D && D->table[0]->segment[0]->text); /* Trailing text present */
if (n_jobs == 0) { /* So not good... */
GMT_Report (API, GMT_MSG_ERROR, "Your time file %s has no records - exiting\n", Ctrl->T.file);
batch_close_files (Ctrl);
Return (GMT_RUNTIME_ERROR);
}
got_time_file = true;
}
/* Get full path to the current working directory */
if (getcwd (topdir, PATH_MAX) == NULL) {
GMT_Report (GMT->parent, GMT_MSG_ERROR, "Unable to determine current working directory - exiting.\n");
batch_close_files (Ctrl);
Return (GMT_RUNTIME_ERROR);
}
gmt_replace_backslash_in_path (topdir);
if (Ctrl->W.active) { /* Do all the work in a temporary directory elsewhere */
if (Ctrl->W.dir) { /* Specified the path to either a relative or absolute temp directory */
#ifdef WIN32
if (Ctrl->W.dir[0] == '/' || Ctrl->W.dir[1] == ':') /* Absolute path under Windows */
#else
if (Ctrl->W.dir[0] == '/') /* Absolute path under *nix|macOS */
#endif
strcpy (workdir, Ctrl->W.dir); /* Use exactly as given */
else /* Gave a relative path */
sprintf (workdir, "%s/%s", topdir, Ctrl->W.dir);
}
else /* Make one in the official temp directory based on N.prefix */
sprintf (workdir, "%s/%s", API->tmp_dir, Ctrl->N.prefix);
}
else /* Use a subdirectory of the current directory */
sprintf (workdir, "%s/%s", topdir, Ctrl->N.prefix);
if (!access ("gmt.conf", R_OK)) { /* User has a gmt.conf file in the top directory that needs to be shared with the jobs */
has_conf = true;
sprintf (conf_file, "%s/gmt.conf", topdir);
gmt_replace_backslash_in_path (conf_file);
if (Ctrl->In.mode == GMT_DOS_MODE) /* Make it suitable for DOS command line copying */
gmt_strrepc (conf_file, '/', '\\');
}
/* Create a working directory which will house every local file and all subdirectories created */
if (gmt_mkdir (workdir)) {
GMT_Report (API, GMT_MSG_ERROR, "An old directory named %s exists OR we were unable to create new working directory %s - exiting.\n", workdir, workdir);
batch_close_files (Ctrl);
Return (GMT_RUNTIME_ERROR);
}
/* Make this directory the current working directory */
if (chdir (workdir)) {
GMT_Report (API, GMT_MSG_ERROR, "Unable to change directory to %s - exiting.\n", workdir);
perror (workdir);
batch_close_files (Ctrl);
Return (GMT_RUNTIME_ERROR);
}
/* Get full path to this working directory */
if (getcwd (cwd, PATH_MAX) == NULL) {
GMT_Report (GMT->parent, GMT_MSG_ERROR, "Unable to determine current working directory.\n");
batch_close_files (Ctrl);
Return (GMT_RUNTIME_ERROR);
}
/* We use DATADIR to include the top and working directory so any files we supply or create can be found while inside job directory */
if (GMT->session.DATADIR) /* Prepend initial and subdir as new datadirs to the existing search list */
sprintf (datadir, "%s,%s,%s", topdir, cwd, GMT->session.DATADIR); /* Start with topdir */
else /* Set the initial and prefix subdirectory as data dirs */
sprintf (datadir, "%s,%s", topdir, cwd);
gmt_replace_backslash_in_path (datadir); /* Since we will be fprintf-ing the path we must use // for a slash */
gmt_replace_backslash_in_path (workdir);
/* Make a path to workdir that works on current architecture (for command line calls) */
strncpy (tmpwpath, workdir, PATH_MAX);
if (Ctrl->In.mode == GMT_DOS_MODE) /* On Windows to do remove a file in a subdir one need to use back slashes */
gmt_strrepc (tmpwpath, '/', '\\');
/* Create the initialization file with settings common to all jobs */
n_written = (n_jobs > 0); /* Know the number of jobs already */
sprintf (init_file, "batch_init.%s", extension[Ctrl->In.mode]);
GMT_Report (API, GMT_MSG_INFORMATION, "Create parameter initiation script %s\n", init_file);
if ((fp = fopen (init_file, "w")) == NULL) {
GMT_Report (API, GMT_MSG_ERROR, "Unable to create file %s - exiting\n", init_file);
batch_close_files (Ctrl);
Return (GMT_ERROR_ON_FOPEN);
}
sprintf (string, "Static parameters set for processing sequence %s", Ctrl->N.prefix);
gmt_set_comment (fp, Ctrl->In.mode, string);
gmt_set_tvalue (fp, Ctrl->In.mode, true, "BATCH_PREFIX", Ctrl->N.prefix);
if (n_written) gmt_set_ivalue (fp, Ctrl->In.mode, false, "BATCH_NJOBS", n_jobs); /* Total jobs (write to init since known) */
if (Ctrl->I.active) { /* Append contents of an include file */
gmt_set_comment (fp, Ctrl->In.mode, "Static parameters set via user include file");
while (gmt_fgets (GMT, line, PATH_MAX, Ctrl->I.fp)) { /* Read the include file and copy to init script with some exceptions */
if (gmt_is_gmtmodule (line, "begin")) continue; /* Skip gmt begin */
if (gmt_is_gmtmodule (line, "end")) continue; /* Skip gmt end */
if (strstr (line, "#!/")) continue; /* Skip any leading shell incantation */
if (strchr (line, '\n') == NULL) strcat (line, "\n"); /* In case the last line misses a newline */
fprintf (fp, "%s", line); /* Just copy the line as is */
}
/* Not closing the include script as we may need to recreate init_file after njobs has been determined */
}
fclose (fp); /* Done writing the init script */
if (Ctrl->S[BATCH_PREFLIGHT].active) { /* Create the preflight script from the user's -Sf script */
/* The preflight script must be modern mode */
sprintf (pre_file, "batch_preflight.%s", extension[Ctrl->In.mode]);
is_classic = gmt_script_is_classic (GMT, Ctrl->S[BATCH_PREFLIGHT].fp);
if (is_classic) {
GMT_Report (API, GMT_MSG_ERROR, "Your preflight file %s is not in GMT modern node - exiting\n", pre_file);
fclose (Ctrl->In.fp);
Return (GMT_RUNTIME_ERROR);
}
GMT_Report (API, GMT_MSG_INFORMATION, "Create preflight script %s and execute it\n", pre_file);
if ((fp = fopen (pre_file, "w")) == NULL) {
GMT_Report (API, GMT_MSG_ERROR, "Unable to create preflight script %s - exiting\n", pre_file);
fclose (Ctrl->In.fp);
Return (GMT_ERROR_ON_FOPEN);
}
gmt_set_script (fp, Ctrl->In.mode); /* Write 1st line of a script */
gmt_set_comment (fp, Ctrl->In.mode, "Preflight script");
fprintf (fp, "%s", export[Ctrl->In.mode]); /* Hardwire a Session Name since sub-shells may mess things up */
if (Ctrl->In.mode == GMT_DOS_MODE) /* Set GMT_SESSION_NAME under Windows to 1 since we run this separately first */
fprintf (fp, "set GMT_SESSION_NAME=1\n");
else /* On UNIX we may use the calling terminal or script's PID as the GMT_SESSION_NAME */
gmt_set_tvalue (fp, Ctrl->In.mode, true, "GMT_SESSION_NAME", "$$");
fprintf (fp, "%s %s\n", load[Ctrl->In.mode], init_file); /* Include the initialization parameters */
while (gmt_fgets (GMT, line, PATH_MAX, Ctrl->S[BATCH_PREFLIGHT].fp)) { /* Read the preflight script and copy to the temporary preflight script with some exceptions */
if (gmt_is_gmtmodule (line, "begin")) { /* Need to insert the DIR_DATA statement */
fprintf (fp, "%s", line);
if (has_conf && !strstr (line, "-C")) fprintf (fp, cpconf[Ctrl->In.mode], conf_file);
fprintf (fp, "\tgmt set DIR_DATA \"%s\"\n", datadir);
}
else if (!strstr (line, "#!/")) { /* Skip any leading shell incantation since already placed by gmt_set_script */
if (strchr (line, '\n') == NULL) strcat (line, "\n"); /* In case the last line misses a newline */
fprintf (fp, "%s", line); /* Just copy the line as is */
}
}
fclose (Ctrl->S[BATCH_PREFLIGHT].fp); /* Done reading the preflight script */
fclose (fp); /* Done writing the temporary preflight script */
#ifndef WIN32 /* Set executable bit if not Windows cmd */
if (chmod (pre_file, S_IRWXU)) {
GMT_Report (API, GMT_MSG_ERROR, "Unable to make preflight script %s executable - exiting.\n", pre_file);
fclose (Ctrl->In.fp);
Return (GMT_RUNTIME_ERROR);
}
#endif
/* Run the pre-flight now which may or may not create a <timefile> needed later via -T, as well as needed data files */
sprintf (cmd, "%s %s", sys_cmd_wait[Ctrl->In.mode], pre_file);
if ((error = system (cmd))) {
GMT_Report (API, GMT_MSG_ERROR, "Running preflight script %s returned error %d - exiting.\n", pre_file, error);
fclose (Ctrl->In.fp);
Return (GMT_RUNTIME_ERROR);
}
}
/* Now we can complete the -T parsing since any given -T<timefile> has now been created by the preflight script */
if (n_jobs == 0) { /* Must check again for a file or decode the argument as a number */
if (!gmt_access (GMT, Ctrl->T.file, R_OK)) { /* A file by that name was indeed created by preflight and is now available */
if ((D = GMT_Read_Data (API, GMT_IS_DATASET, GMT_IS_FILE, GMT_IS_NONE, GMT_READ_NORMAL, NULL, Ctrl->T.file, NULL)) == NULL) {
GMT_Report (API, GMT_MSG_ERROR, "Unable to read time file: %s - exiting\n", Ctrl->T.file);
fclose (Ctrl->In.fp);
Return (API->error);
}
if (D->n_segments > 1) { /* We insist on a simple file structure with a single segment */
GMT_Report (API, GMT_MSG_ERROR, "Your time file %s has more than one segment - reformat first\n", Ctrl->T.file);
fclose (Ctrl->In.fp);
Return (API->error);
}
n_jobs = (unsigned int)D->n_records; /* Number of records means number of jobs */
n_values = (unsigned int)D->n_columns; /* The number of per-job parameters we need to place into the per-job parameter files */
has_text = (D && D->table[0]->segment[0]->text); /* Trailing text present */
got_time_file = true;
}
else if (gmt_count_char (GMT, Ctrl->T.file, '/') == 2) { /* Give a vector specification -Tmin/max/inc, call gmtmath to build the array */
char output[GMT_VF_LEN] = {""}, cmd[GMT_LEN128] = {""};
unsigned int V = GMT->current.setting.verbose;
if (GMT_Open_VirtualFile (API, GMT_IS_DATASET, GMT_IS_NONE, GMT_OUT|GMT_IS_REFERENCE, NULL, output) == GMT_NOTSET) {
Return (API->error);
}
if (GMT->common.f.active[GMT_IN])
sprintf (cmd, "-T%s -o1 -f%s --GMT_HISTORY=readonly T = %s", Ctrl->T.file, GMT->common.f.string, output);
else
sprintf (cmd, "-T%s -o1 --GMT_HISTORY=readonly T = %s", Ctrl->T.file, output);
GMT_Report (API, GMT_MSG_INFORMATION, "Calling gmtmath with args %s\n", cmd);
GMT->current.setting.verbose = GMT_MSG_ERROR; /* So we don't get unwanted verbosity from gmtmath */
if (GMT_Call_Module (API, "gmtmath", GMT_MODULE_CMD, cmd)) {
Return (API->error); /* Some sort of failure */
}
GMT->current.setting.verbose = V; /* Restore */
if ((D = GMT_Read_VirtualFile (API, output)) == NULL) { /* Load in the data array */
Return (API->error); /* Some sort of failure */
}
n_jobs = (unsigned int)D->n_records; /* Number of records means number of jobs */
n_values = (unsigned int)D->n_columns; /* The number of per-job parameters we need to place into the per-job parameter files */
has_text = (D && D->table[0]->segment[0]->text); /* Trailing text present */
if (strchr (Ctrl->T.file, 'T')) /* Check here since gmtmath does not pass that info back */
gmt_set_column_type (GMT, GMT_IN, GMT_X, GMT_IS_ABSTIME); /* Set first input column type as absolute time */
}
else { /* Just gave the number of jobs (we hope, or we got a bad filename and atoi should return 0) */
n_jobs = atoi (Ctrl->T.file);
issue_col0_par = true;
}
}
if (n_jobs == 0) { /* So not good... */
GMT_Report (API, GMT_MSG_ERROR, "No jobs specified! - exiting.\n");
fclose (Ctrl->In.fp);
Return (GMT_RUNTIME_ERROR);
}
if (Ctrl->F.active) {
if (!got_time_file) {
GMT_Report (API, GMT_MSG_ERROR, "Option -F: Requires a file via the -T option - exiting.\n");
fclose (Ctrl->In.fp);
Return (GMT_RUNTIME_ERROR);
}
n_fmts = gmt_count_char (GMT, Ctrl->F.template, '%');
if (strstr (Ctrl->F.template, "%s")) {
n_fmts--; /* One less since trailing text will be used */
if (D->type == GMT_READ_DATA) {
GMT_Report (API, GMT_MSG_ERROR, "Option -F: A string is expected by the template but your -T file has no trailing text.\n");
fclose (Ctrl->In.fp);
Return (GMT_RUNTIME_ERROR);
}
}
else if (n_fmts == 0) {
GMT_Report (API, GMT_MSG_ERROR, "Option -F: Your template has no format statements so cannot be used.\n");
fclose (Ctrl->In.fp);
Return (GMT_RUNTIME_ERROR);
}
if (n_fmts > D->n_columns) {
GMT_Report (API, GMT_MSG_ERROR, "Option -F: Not enough input columns in your -T file to satisfy template.\n");
fclose (Ctrl->In.fp);
Return (GMT_RUNTIME_ERROR);
}
}
if (!n_written) { /* Rewrite the init file to place the BATCH_NJOBS there */
GMT_Report (API, GMT_MSG_INFORMATION, "Recreate parameter initiation script given njobs has been set %s\n", init_file);
(void) gmt_remove_file (GMT, init_file); /* Delete the first init file */
if ((fp = fopen (init_file, "w")) == NULL) {
GMT_Report (API, GMT_MSG_ERROR, "Unable to create file %s - exiting\n", init_file);
batch_close_files (Ctrl);
Return (GMT_ERROR_ON_FOPEN);
}
sprintf (string, "Static parameters set for processing sequence %s", Ctrl->N.prefix);
gmt_set_comment (fp, Ctrl->In.mode, string);
gmt_set_tvalue (fp, Ctrl->In.mode, true, "BATCH_PREFIX", Ctrl->N.prefix);
gmt_set_ivalue (fp, Ctrl->In.mode, false, "BATCH_NJOBS", n_jobs); /* Total jobs (write to init since known) */
if (Ctrl->I.active) { /* Append contents of an include file */
rewind (Ctrl->I.fp); /* Reposition to start of this open file */
gmt_set_comment (fp, Ctrl->In.mode, "Static parameters set via user include file");
while (gmt_fgets (GMT, line, PATH_MAX, Ctrl->I.fp)) { /* Read the include file and copy to init script with some exceptions */
if (gmt_is_gmtmodule (line, "begin")) continue; /* Skip gmt begin */
if (gmt_is_gmtmodule (line, "end")) continue; /* Skip gmt end */
if (strstr (line, "#!/")) continue; /* Skip any leading shell incantation */
if (strchr (line, '\n') == NULL) strcat (line, "\n"); /* In case the last line misses a newline */
fprintf (fp, "%s", line); /* Just copy the line as is */
}
}
fclose (fp); /* Done writing the init script */
}
if (Ctrl->I.active) fclose (Ctrl->I.fp); /* Done reading the include script */
if (Ctrl->In.mode == GMT_DOS_MODE)
gmt_strrepc (topdir, '/', '\\'); /* Temporarily make DOS compatible */
n_to_run = (Ctrl->M.active) ? 1 : n_jobs;
GMT_Report (API, GMT_MSG_INFORMATION, "Number of main processing jobs: %d\n", n_to_run);
if (Ctrl->T.precision) /* Precision was prescribed */
precision = Ctrl->T.precision;
else /* Compute width from largest job number */
precision = irint (ceil (log10 ((double)(Ctrl->T.start_job+n_jobs-1))+0.1)); /* Width needed to hold largest job number, guaranteed to give at least 1 */
/* Create parameter include files, one for each job */
GMT_Report (API, GMT_MSG_INFORMATION, "Parameter files for main processing: %d\n", n_to_run);
for (i_job = 0; i_job < n_jobs; i_job++) {
job = data_job = i_job + Ctrl->T.start_job; /* The job is normally same as data_job number */
if (Ctrl->M.active && job != Ctrl->M.job) continue; /* Just doing a single job for debugging */
sprintf (state_tag, "%*.*d", precision, precision, job);
sprintf (state_prefix, "batch_params_%s", state_tag);
sprintf (param_file, "%s.%s", state_prefix, extension[Ctrl->In.mode]);
if ((fp = fopen (param_file, "w")) == NULL) {
GMT_Report (API, GMT_MSG_ERROR, "Unable to create job parameter file %s - exiting\n", param_file);
fclose (Ctrl->In.fp);
Return (GMT_ERROR_ON_FOPEN);
}
sprintf (state_prefix, "Parameter file for job %s", state_tag);
gmt_set_comment (fp, Ctrl->In.mode, state_prefix);
sprintf (state_prefix, "%s_%s", Ctrl->N.prefix, state_tag);
gmt_set_tvalue (fp, Ctrl->In.mode, false, "BATCH_DIR", state_prefix); /* Current directory name for the job (e.g., my_job_0003) */
if (Ctrl->F.active) { /* Build product name from input table and template */
char custom_name[GMT_LEN512] = {""};
batch_set_product_prefix (GMT, D, data_job, n_fmts, Ctrl->F.template, custom_name);
gmt_set_tvalue (fp, Ctrl->In.mode, false, "BATCH_NAME", custom_name); /* Custom product name prefix via template */
}
else /* Default current product name prefix (e.g., my_job_0003) */
gmt_set_tvalue (fp, Ctrl->In.mode, false, "BATCH_NAME", state_prefix);
gmt_set_ivalue (fp, Ctrl->In.mode, false, "BATCH_JOB", data_job); /* Current job number (e.g., 3) */
gmt_set_tvalue (fp, Ctrl->In.mode, false, "BATCH_ITEM", state_tag); /* Current job tag (formatted job number, e.g, 0003) */
for (col = 0; col < n_values; col++) { /* Derive job variables from this row in <timefile> and copy to each parameter file as script variables */
sprintf (string, "BATCH_COL%u", col);
gmt_set_value (GMT, fp, Ctrl->In.mode, col, string, D->table[0]->segment[0]->data[col][data_job]);
}
if (issue_col0_par) gmt_set_ivalue (fp, Ctrl->In.mode, false, "BATCH_COL0", data_job); /* Same as current job number */
if (has_text) { /* Also place any string parameter as a single string variable */
gmt_set_tvalue (fp, Ctrl->In.mode, false, "BATCH_TEXT", D->table[0]->segment[0]->text[data_job]);
if (Ctrl->T.split) { /* Also split the string into individual words BATCH_WORD1, BATCH_WORD2, etc. */
char *word = NULL, *trail = NULL, *orig = strdup (D->table[0]->segment[0]->text[data_job]);
col = 0;
trail = orig;
while ((word = strsep (&trail, Ctrl->T.sep)) != NULL) {
if (*word != '\0') { /* Skip empty strings */
sprintf (string, "BATCH_WORD%u", col++);
gmt_set_tvalue (fp, Ctrl->In.mode, false, string, word);
}
}
gmt_M_str_free (orig);
}
}
fclose (fp); /* Done writing this parameter file */
}
/* Now build the main loop script from the mainscript */
sprintf (main_file, "batch_job.%s", extension[Ctrl->In.mode]);
GMT_Report (API, GMT_MSG_INFORMATION, "Create main batch job script %s\n", main_file);
if ((fp = fopen (main_file, "w")) == NULL) {
GMT_Report (API, GMT_MSG_ERROR, "Unable to create loop job script file %s - exiting\n", main_file);
fclose (Ctrl->In.fp);
Return (GMT_ERROR_ON_FOPEN);
}
gmt_set_script (fp, Ctrl->In.mode); /* Write 1st line of a script */
gmt_set_comment (fp, Ctrl->In.mode, "Main job loop script");
fprintf (fp, "%s", export[Ctrl->In.mode]); /* Hardwire a GMT_SESSION_NAME since sub-shells may mess things up */
if (Ctrl->In.mode == GMT_DOS_MODE) /* Set GMT_SESSION_NAME under Windows to be the job number */
fprintf (fp, "set GMT_SESSION_NAME=%c1\n", var_token[Ctrl->In.mode]);
else /* On UNIX we use the script's PID as GMT_SESSION_NAME */
gmt_set_tvalue (fp, Ctrl->In.mode, true, "GMT_SESSION_NAME", "$$");
gmt_set_comment (fp, Ctrl->In.mode, "Include static and job-specific parameters");
fprintf (fp, "%s %s\n", load[Ctrl->In.mode], init_file); /* Include the initialization parameters */
fprintf (fp, "%s batch_params_%c1.%s\n", load[Ctrl->In.mode], var_token[Ctrl->In.mode], extension[Ctrl->In.mode]); /* Include the job parameters */
fprintf (fp, "mkdir %s\n", gmt_place_var (Ctrl->In.mode, "BATCH_DIR")); /* Make a temp directory for this job */
fprintf (fp, "cd %s\n", gmt_place_var (Ctrl->In.mode, "BATCH_DIR")); /* cd to the temp directory */
while (gmt_fgets (GMT, line, PATH_MAX, Ctrl->In.fp)) { /* Read the main script and copy to loop script, with some exceptions */
if (gmt_is_gmtmodule (line, "begin")) { /* Must insert DIR_DATA setting */
fprintf (fp, "%s", line);
if (has_conf && !strstr (line, "-C")) fprintf (fp, cpconf[Ctrl->In.mode], conf_file);
fprintf (fp, "\tgmt set DIR_DATA \"%s\"\n", datadir);
}
else if (!strstr (line, "#!/")) { /* Skip any leading shell incantation since already placed */
if (strchr (line, '\n') == NULL) strcat (line, "\n"); /* In case the last line misses a newline */
fprintf (fp, "%s", line); /* Just copy the line as is */
}
}
fclose (Ctrl->In.fp); /* Done reading the main script */
if (!Ctrl->D.active) /* Move job products up to main directory */
fprintf (fp, "%s %s* %s\n", mvfile[Ctrl->In.mode], gmt_place_var (Ctrl->In.mode, "BATCH_NAME"), topdir);
fprintf (fp, "cd ..\n"); /* cd up to parent dir */
/* Create completion file so batch knows this job is done */
fprintf (fp, "%s %s.___\n", createfile[Ctrl->In.mode], gmt_place_var (Ctrl->In.mode, "BATCH_DIR"));
if (!Ctrl->Q.active) { /* Delete evidence; otherwise we want to leave debug evidence when doing a single job only */
gmt_set_comment (fp, Ctrl->In.mode, "Remove job directory and job parameter file");
fprintf (fp, "%s %s\n", rmdir[Ctrl->In.mode], gmt_place_var (Ctrl->In.mode, "BATCH_DIR")); /* Remove the work dir and any files in it */
fprintf (fp, "%s batch_params_%c1.%s\n", rmfile[Ctrl->In.mode], var_token[Ctrl->In.mode], extension[Ctrl->In.mode]); /* Remove the parameter file for this job */
}
if (Ctrl->In.mode == GMT_DOS_MODE) /* This is crucial to the "start /B ..." statement below to ensure the DOS process terminates */
fprintf (fp, "exit\n");
fclose (fp); /* Done writing loop script */
if (Ctrl->In.mode == GMT_DOS_MODE)
gmt_strrepc (topdir, '\\', '/'); /* Revert */
#ifndef WIN32 /* Set executable bit if not Windows cmd */
if (chmod (main_file, S_IRWXU)) {
GMT_Report (API, GMT_MSG_ERROR, "Unable to make script %s executable - exiting\n", main_file);
Return (GMT_RUNTIME_ERROR);
}
#endif
GMT_Report (API, GMT_MSG_INFORMATION, "Total jobs to process: %u\n", n_to_run);
if (Ctrl->Q.scripts) { /* No processing executed */
Return (GMT_NOERROR); /* We are done */
}
if (Ctrl->M.active) { /* Just run that one job */
sprintf (cmd, "%s %s %s", sys_cmd_wait[Ctrl->In.mode], main_file, state_tag);
GMT_Report (API, GMT_MSG_INFORMATION, "Run master script %s %s\n", main_file, state_tag);
if ((error = run_script (cmd))) {
GMT_Report (API, GMT_MSG_ERROR, "Running master script %s for argument %s returned error %d - exiting.\n", main_file, state_tag, error);
Return (GMT_RUNTIME_ERROR);
}
Return (GMT_NOERROR); /* We are done */
}
/* Finally, we can run all the jobs in a controlled loop, launching new parallel jobs as cores become available */
i_job = first_i_job = 0; n_jobs_not_started = n_jobs;
job = Ctrl->T.start_job;
n_cores_unused = MAX (1, Ctrl->x.n_threads - 1); /* Save one core for the main batch module thread */
status = gmt_M_memory (GMT, NULL, n_jobs, struct BATCH_STATUS); /* Used to keep track of job status */
GMT_Report (API, GMT_MSG_INFORMATION, "Build jobs using %u cores\n", n_cores_unused);
/* START PARALLEL EXECUTION OF JOB SCRIPTS */
GMT_Report (API, GMT_MSG_INFORMATION, "Execute batch job scripts in parallel\n");
while (!done) { /* Keep running jobs until all jobs have completed */
while (n_jobs_not_started && n_cores_unused) { /* Launch new jobs if possible */
#ifdef WIN32
if (Ctrl->In.mode < GMT_DOS_MODE) /* A bash or csh run from Windows. Need to call via "start" to get parallel */
sprintf (cmd, "start /B %s %s %*.*d", sys_cmd_nowait[Ctrl->In.mode], main_file, precision, precision, job);
else /* Running batch, so no need for the above trick */
sprintf (cmd, "%s %s %*.*d &", sys_cmd_nowait[Ctrl->In.mode], main_file, precision, precision, job);
#else
sprintf (cmd, "%s %s %*.*d &", sys_cmd_nowait[Ctrl->In.mode], main_file, precision, precision, job);
#endif
GMT_Report (API, GMT_MSG_DEBUG, "Launch script for job %*.*d\n", precision, precision, job);
if ((error = system (cmd))) {
GMT_Report (API, GMT_MSG_ERROR, "Running script %s returned error %d - aborting.\n", cmd, error);
Return (GMT_RUNTIME_ERROR);
}
status[i_job].started = true; /* We have now launched this job job */
job++; /* Advance to next job for next launch */
i_job++; /* Advance to next job for next launch */
n_jobs_not_started--; /* One less job remaining */
n_cores_unused--; /* This core is now busy */
}
gmt_sleep (BATCH_WAIT_TO_CHECK); /* Wait 0.01 second - then check for the completion file */
for (k = first_i_job; k < i_job; k++) { /* Only loop over the range of jobs that we know are currently in play */
if (status[k].completed) continue; /* Already finished with this job */
if (!status[k].started) continue; /* Not started this job yet */
/* Here we can check if the job job has completed by looking for the completion file */
sprintf (completion_file, "%s_%*.*d.___", Ctrl->N.prefix, precision, precision, Ctrl->T.start_job+k);
if (access (completion_file, F_OK)) continue; /* Not found yet */
n_jobs_completed++; /* One more job completed */
status[k].completed = true; /* Flag this job as completed */
n_cores_unused++; /* Free up the core */
percent = 100.0 * n_jobs_completed / n_jobs;
(void) gmt_remove_file (GMT, completion_file); /* Delete the completion file */
GMT_Report (API, GMT_MSG_INFORMATION, "Job %*.*d of %d completed [%5.1f %%]\n", precision, precision, k+1, n_jobs, percent);
}
/* Adjust first_i_job, if needed */
while (first_i_job < n_jobs && status[first_i_job].completed) first_i_job++;
if (n_jobs_completed == n_jobs) done = true; /* All jobs completed! */
}
/* END PARALLEL EXECUTION OF JOB SCRIPTS */
gmt_M_free (GMT, status); /* Done with this structure array */
P_len = strlen (Ctrl->N.prefix); /* Length of the prefix */
if (Ctrl->S[BATCH_POSTFLIGHT].active) { /* Prepare the temporary postflight script */