All, The previous file I sent out contains a bug I introduced while testing the script. Please use the corrected version which is attached. (Lesson learned -- always test your fix with another dataset!) -- Jon Jonathan Callahan wrote: All, |
\cancel mode verify ! std_list - create a listing, CDF or ASCII ! 10/94 2/95 ! REQUIRES Ferret V5.22 or better ! 7/00 *sh*: use REPEAT to support large outputs ! - removed mode CALENDAR:DAYS ! Description: create a Y listing, CDF or ASCII ! Usage: GO std_list data_set variable output_file_name cdf_or_asc ! argument 4, cdf_or_asc, may have values "txt", "cdf", "tsv", "csv", or "asc" ! check arguments query/ignore "$1"%<Usage: GO std_list data_set variable output_file_name cdf_or_asc% query/ignore $2%<Usage: GO std_list data_set variable output_file_name cdf_or_asc% ! hack: arg 3 comes in with a double slash which upsets the Ferret parser !query/ignore args $3%<Usage: GO std_list data_set variable output_file_name cdf_or_asc% query/ignore $4%|txt|cdf|tsv|csv|asc|% ! decode format argument define symbol format "$4%|txt>/heading/width=1024|cdf>/format=cdf|tsv>/format=tab|csv>/format=comma|asc>/heading=enhanced/width=1024/format=(5G12.5)|%" cancel mode interpolate ! important when evaluating the size limits set list/file="$3" ! NOTE: TO ACHIEVE LISTINGS IN EXCESS OF FERRET MEM SIZE THIS SCRIPT REQUIRES ! THAT THE DATA SIZE CAN BE OBTAINED WITHOUT FULL EVALUATION OF THE RESULT. ! THIS WILL NOT BE THE CASE IF ! o AN AXIS OF THE "LET" DEFINITION OF THE VAR TO BE LISTED CONTAINS ! POTENTIAL DATA DEPENDENCIES (E.G. A GRID-CHANGING ! FUNCTION THAT RETURNS AN ABSTRACT OR NEW AXIS SUCH AS FFTA() ) ! IN THOSE CASES YOU MUST DEFINE THE FERRET VARIABLE ($VAR_GRID) ! VERSION OF THIS SCRIPT IN WHICH YOU SPECIFY A FILE VARIABLE OR KNOWN ! GRID AS THE ARGUMENT OF SET GRID ! Aside: this is an imperfect solution for these cases ... needs exploring IF ($VAR_GRID"0") THEN set grid ($VAR_GRID) ELSE set grid $2[d=$1] ENDIF define symbol theVar $2[d=$1] ! For requests that encompass 360 deg of longitude Ferret returns N points if ! the range boundaries lie exactly on grid cell boundaries and N+1 points ! otherwise. Folks requesting data downloads don't want to have their their ! first data point (sometimes) repeated so we test for that here and switch ! to index space. define symbol user_istart `($theVar),return=istart` define symbol user_iend `($theVar),return=iend` can region/x define symbol data_isize `($theVar),return=isize` IF `($user_iend) - ($user_istart) + 1 - ($data_isize) GT 0` THEN define symbol new_iend `($user_iend) - 1` ELSE define symbol new_iend `($user_iend)` ENDIF set region/i=($user_istart):($new_iend) ! Now get the start, stop and size information define symbol isize `($theVar),return=isize` define symbol jmin `($theVar),return=jstart` define symbol jmax `($theVar),return=jend` define symbol jsize `($theVar),return=jsize` define symbol kmin `($theVar),return=kstart` define symbol kmax `($theVar),return=kend` define symbol ksize `($theVar),return=ksize` define symbol lmin `($theVar),return=lstart` define symbol lmax `($theVar),return=lend` define symbol lsize `($theVar),return=lsize` ! return=*start returns 0 if the axis is an undefined axis ! This messes messes up the repeat loop below if the data is chunked. ! A few more lines of code to handle this. if `($jmin) lt 1` then define symbol jmin 1 if `($jmax) lt 1` then define symbol jmax 1 if `($kmin) lt 1` then define symbol kmin 1 if `($kmax) lt 1` then define symbol kmax 1 if `($lmin) lt 1` then define symbol lmin 1 if `($lmax) lt 1` then define symbol lmax 1 let size = ($isize)*($jsize)*($ksize)*($lsize) ! default chunk sizes let jchunk = ($jsize) ! dflt size = full span let kchunk = ($ksize) let lchunk = INT(max_size/size * ($lsize)) ! if mode interpolate is desired ! **MODE INTERPOLATE NEEDS TO BE CONTROLLED IN A UNIFORM MANNER FOR ALL SCRIPTS !set mode interpolate ! the default is set under 1/4 of Ferret's memory allowing for LET definitions let max_size = ($MAX_LIST_SIZE"1000000") ! use this for debugging, too IF `size LE max_size` THEN ! ... list in one big chunk ! This special case is not strictly necessary -- the logic for listing ! large files also works for small ... but with some overhead list/append/file($format) ($theVar) ELSE ! determine how to break the listing into small chunks ! chunk along L axis. If still too big then along K axis, too. If still ... IF `lchunk EQ 0` THEN let lchunk = 1 let kchunk = INT(max_size/(size/($lsize)) * ($ksize)) IF `kchunk EQ 0` THEN let kchunk = 1 let jchunk = INT(max_size/(size/(($lsize)*($ksize))) * ($jsize) ) IF `jchunk EQ 0` THEN let jchunk = 1 ! always list a full span in I ... possibly exceeding max_size ENDIF ENDIF ENDIF let jtop = MIN(j+jchunk-1, ($jmax) ) let ktop = MIN(k+kchunk-1, ($kmax) ) let ltop = MIN(l+lchunk-1, ($lmax) ) IF $4"0|cdf>1|*>0" THEN ! ... netCDF coutput in chunks REPEAT/l=($lmin):($lmax):`lchunk` REPEAT/k=($kmin):($kmax):`kchunk` REPEAT/j=($jmin):($jmax):`jchunk` (list/append/file($format)/j=`j`:`jtop`/k=`k`:`ktop`/l=`l`:`ltop`/jlimits=($jmin):($jmax)/klimits=($kmin):($kmax)/llimits=($lmin):($lmax) ($theVar); CANC MEM/ALL) ELSE ! ... ASCII output ! ... Note that large ASCII output as-is is highly imperfect because header ! information gets embedded into the data ! Proper solution of this problem requires us to know which is the #2 ! axis of the output (e.g. in an XYZT output the Y axis is #2 by dflt) ! and forbid chunking along either axis of this plane -- REPEAT/l=($lmin):($lmax):`lchunk` REPEAT/k=($kmin):($kmax):`kchunk` REPEAT/j=($jmin):($jmax):`jchunk` (list/append/file($format)/j=`j`:`jtop`/k=`k`:`ktop`/l=`l`:`ltop` ($theVar); CANC MEM/ALL) ENDIF ENDIF