// Using AstroImageJ Data Processor in Batch Processing Mode
// This is based on my requirements but shows how the basic approach works

// STAGE ONE
// In my case I need to collect a list of directories containing the data I want to process

// Root/StarName/Date/Filter/Gain/*.fit
// the directory Calibrated is built below this i.e Root/StarName/Date/Filter/Gain/Calibrated

// Recursively process the folders in a user-specified directory

  requires("1.47i");
  dir = getDirectory("Choose Root Directory ");
 
 // This function gets all folders from a folder 

	myFolderList = getFolderList(dir);   // Star Names

	myFolderList2 = newArray(0);	// define list of Star Names and Dates	
	for (i=0;i< myFolderList.length;i++){
		nextList = getFolderList(myFolderList[i]);
		myFolderList2 = Array.concat(myFolderList2,nextList);   // Dates
	}

	myFolderList3 = newArray(0);	// define list of Star Names and Dates and Filters
	for (i=0;i< myFolderList2.length;i++){
		nextList = getFolderList(myFolderList2[i]);   
            	myFolderList3 = Array.concat(myFolderList3,nextList);   // Filters
	}

	myFolderList4 = newArray(0); // define list of Star Names and Dates and Filters and Gain
	for (i=0;i< myFolderList3.length;i++){
		nextList = getFolderList(myFolderList3[i]);   
            	myFolderList4 = Array.concat(myFolderList4,nextList);   // Gain
	}


// myFolderList4 contains all the directories with data to input the Data Processor

// STAGE TWO

// MAIN Processing Loop
	BiasDarkDate = "\\2019-11-19\\";  // fixed long intervals between builds
	Processed = 0;
	print("Number of Potential Data Folders = " + myFolderList4.length);

	for (kk = 0; kk < myFolderList4.length; kk++){
		jj = kk + 1;
		print("Folder " + jj + " of " + myFolderList4.length);
// before using each entry do a getFolderList and check for another level which might be Calibrated 
// this in general would indicate it is already processed so ignore 

	myFolderList5 = getFolderList(myFolderList4[kk]);  
	if(myFolderList5.length == 0){ // this directory has not been processed!!
			Processed = Processed + 1;
// parse each entry to extract date,gain and filter so that the directory for the master bias,darks and flats
// can be set up as well - all these will be used to set input for the Data Processor

			DirEntry = myFolderList4[kk];
			print("Directory to Process " + DirEntry);
			sdate = indexOf(DirEntry,"/20");   // the dates are of the form "20xx-mm-dd"
			Fdate = substring(DirEntry,sdate+1,sdate+11);
			print("Date of files - " + Fdate);
			gindex = indexOf(DirEntry,"ain");   // looking for Gain or gain part of directory
			Gain = "G" + substring(DirEntry,gindex); 
			dir = substring(DirEntry,0,gindex-4); // this will form part of the post processing stacking
			
			print("Gain of Files "+ Gain);
			Filter = substring(DirEntry,gindex -4,gindex-1);
			print("Filter for files " + Filter);  // this the filter used for these images

// for my purposes I use a previously built master bias and darks in known directories 
// but they are tied to different camera gains and the darks are processed as scalable
// here the Gain extracted from the data directory determines the gain subdirectory of the master bias and darks
// if it doesn't exist - very uncommon but will happen with new bias and darks - change BiasDark Date above


			MasterBias = "X:\\Master\\Bias\\" + Gain + BiasDarkDate;
//  IMPORTANT this root directory structure is prebuilt - further directories are built below this level
			BiasDir="X\:\\Bias\\" + BiasDarkDate + Gain;
			if(File.isDirectory(MasterBias)){
			// here we have already master bias set up
				call("ij.Prefs.set","dataproc.createBias","false");
						
				}
				else
				{
			// here we have to build the master bias as part of the Data Processor run
			// make sure the file pattern is preset to *.fit
				call("ij.Prefs.set","dataproc.createBias","true");
				call("ij.Prefs.set","dataproc.biasRawDirText",BiasDir);
				MasterBias = "X\:\\Master\\Bias\\"+ Gain + BiasDarkDate;
				File.makeDirectory(MasterBias); 
				
				}
			
			
			
			MasterDark = "X:\\Master\\Darks\\" + Gain + BiasDarkDate;
//  IMPORTANT this root directory structure is prebuilt - further directories are built below this level
			DarkDir="X\:\\Dark\\" + BiasDarkDate + Gain;
			if(File.isDirectory(MasterDark)){
			// here we have already master bias set up
				call("ij.Prefs.set","dataproc.createDark","false");
						
				}
				else
				{
			// here we have to build the master dark as part of the Data Processor run
			// make sure the file pattern is preset to *.fit
				call("ij.Prefs.set","dataproc.createDark","true");
				call("ij.Prefs.set","dataproc.darkRawDirText",DarkDir);
				MasterDark = "X\:\\Master\\Darks\\"+ Gain + BiasDarkDate;
				File.makeDirectory(MasterDark); 
				
				}

// here the master flat name is constructed in the same way but includes date of the flat (same as images) and filter used
// I always use the gain 0 as I have flat panel I use before a night of image capture

			MasterFlats = "X\:\\Master\\Flats\\" + Fdate + Filter;
//  IMPORTANT this root directory structure is prebuilt - further directories are built below this level
// this is directory where the raw flats images are stored for building
			FlatDir = "X:\\Flats\\" + Fdate + Filter + "gain_0";

// does this master flat exist it only needs to be built for the first use on this date and filter
// check for existence - if not then set up build

// at this point we start to configure Data Processor parameters which are stored in a file AIJ_Prefs.txt
// here we set a new parameter especially for batch processing which needs latest build of AstroImageJ 3.3.1.08+ Thank you Karen!

			call("ij.Prefs.set","dataproc.autoRunAndClose","true");  
 
			if(File.isDirectory(MasterFlats)){
			// here we have already master flats set up
				call("ij.Prefs.set","dataproc.createFlat","false");
						
				}
				else
				{
			// here we have to build the master flat as part of the Data Processor run
			// make sure the file pattern is preset to *.fit
				call("ij.Prefs.set","dataproc.createFlat","true");
				call("ij.Prefs.set","dataproc.flatRawDirText",FlatDir);
				MasterFlats = "X\:\\Master\\Flats\\" + Fdate;
				File.makeDirectory(MasterFlats); // need to make directory in stages!
				MasterFlats = "X\:\\Master\\Flats\\" + Fdate + Filter;
				File.makeDirectory(MasterFlats); 
				}
				
			call("ij.Prefs.set","dataproc.flatMasterDirText",MasterFlats);
			call("ij.Prefs.set","dataproc.biasMasterDirText",MasterBias);
			call("ij.Prefs.set","dataproc.darkMasterDirText",MasterDark);
			call("ij.Prefs.set","dataproc.mainDir",DirEntry);
			list = getFileList(DirEntry);
			Count = list.length;  // this used to check all files have been added to Calibrated sub folder
			CalEntry = DirEntry + "/Calibrated/";
// here we start the Data Processor and it will immediately start running and will close when finished
// All DP parameters not set here will be same as the last run (read in from AIJ_Prefs.txt file!)
// IMPORTANT that you are happy with all these settings - General, Platesolve etc. 
 		
			run("Data Processor");
// WARNING this is running aysnchronously - so we have to set up a wait loop


			waitFlag = true;
			
// now we go into wait state for Data Processor to run
			
			while(waitFlag){
				wait(10000);
			// check for Calibrated directory - the results directory
			// the directory exists check for how many files have been written cf the starting number in the data directory	
				if(File.isDirectory(CalEntry)){ 
					list = getFileList(CalEntry);
					if(list.length >= Count) waitFlag = false;
				}
			}; // end of while

// we now have all the images processed by DP in this directory

// what follows is the processing for stacking the images and writing out stack result with a modified FITS Header into the
// Root/Starname/  directory with the file name Date+Filter.fit which I then analyse with AstroImageJ by hand using aperture files etc.
 
// now ready to run version of StackAlign

 			
 			wait(10000);   // just make sure we are completely finished DP
  			file = CalEntry + list[0];
  			i = list.length;
// IMPORTANT - check the number of images you can stack in memory as defined by the startup of AstroImageJ
// this number is a comfortable maximum images for loading into AstroimageJ on my machine and its memory setting (12 GB)
			if(i < 100){			
 			print ("Loading Sequence");
  			run("Image Sequence...", "open=&file number=i starting=1 increment=1 scale=100 file=[] or=[] sort");
		// I have preset that stack aligning will be done by WCS
  			run("Align Stack", "first=1 last=i radius=8 inner=14 outer=21 use show");
			print("Waiting for Align stack to complete");
  			while (isOpen("Aligned_") == false){	// again we have an asynchronous process we have to wait for 
			wait(5000);
            		print("waiting...");
     			 };
 			selectWindow("Aligned_");
 			midslice = nSlices >> 1;	// find the mid image of stack - we will use this for its FITS Header (mid time of stack exposures)
 			Nos = nSlices;
  			setSlice(midslice);		// set this mid image as the reference for FITS Header copy
  			print("Averaging Stack");
  			run("Z Project...", "start=1 stop=i projection=[Average Intensity]");
 			print("Copying FITS header");
 			run("Copy FITS Header", "from=Aligned_ to=AVG_Aligned_ history=[Averaged Intensity of stack]");
  			wait(1000);
  			selectWindow("AVG_Aligned_"); // this is stacked image we are going use for the final result
			// extracting the FITS Header info
 			 lines = split(getInfo(),'\n');
  			eline = 0;
  			for (i=0;i<lines.length;i++){
				if(startsWith(lines[i],"EXPOSURE")){   // looking for the entry EXPOSURE - this will depend on your data
				eline = i;
               
        			 	}
  			 }
			
   			if(eline == 0){
			oexp = 25;
			print("No Exposure found - default to 25 secs"); 
			}
       			 else
			{
        // Now extract exposure string
			header = split(lines[eline]);
       			oexp = header[1];
        			oexp = Nos * oexp;
       			}
			ioexp = d2s(oexp,0);

  			print ("Adjusting Exposure to "+ ioexp);
  			run("Write FITS Header", "image=AVG_Aligned_ type=[integer] fits=EXPOSURE value=&ioexp comment=[Exposure time in seconds]");
			// FITS Header has been modified with the total exposure of the stack

	// Currently I have three filter possibilities in my data and I handle this simply as the following 
  			
			if(Filter == "/V/"){
  				file = dir + "V.fits";
			}
			
			if(Filter == "/R/"){
  				
  				file = dir + "R.fits";
			}
			
			if(Filter == "/B/"){
  				
  				file = dir + "B.fits";
			}
  			print ( "saving... "+file);
  			save(file);
  			print("File Saved");

			run("Close All");     // this make sure all stack images are released from storage ready for next loop over data directories
 			} // end of i < 100
			else
			{
			print("Stack not processed - too many files");   // this covers a possible memory overload which causes failures
			run("Close All");
			}
		} // end of if unprocessed directory
		else
		{
		DirEntry = myFolderList4[kk];
		print("Directory Ignored - already Processed " + DirEntry);
		}
	} // end of main Loop

print("Number of Folders processed this run " + Processed);
print("*****End of BatchDP*****");

// this is the function for extracting folders from a path in stage ONE above

function getFolderList(inputPathFunction) 
{ 	
       fileListFunction = getFileList(inputPathFunction);  //read file list 
       Array.sort(fileListFunction); 
       returnedFileList = newArray(0);     //this list stores all found folders on this path and is returned at the end of the function 
      	for (j=0; j < fileListFunction.length; j++) { 
 	 	if (endsWith(fileListFunction[j], "/")){
		returnedFileList = Array.concat(returnedFileList,"" + inputPathFunction + fileListFunction[j]);      //if it is a folder 

		}
	} 
      return returnedFileList; 
 }