/***************************************************************
 *
 * Copyright (C) 1990-2007, Condor Team, Computer Sciences Department,
 * University of Wisconsin-Madison, WI.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License.  You may
 * obtain a copy of the License at
 * 
 *    http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 ***************************************************************/


#include "condor_common.h"
#include "condor_classad.h"
#include "condor_attributes.h"
#include "condor_debug.h"
#include "condor_io.h"
#include "condor_uid.h"
#include "shadow.h"
#include "event_notification.h"
#include "guidance.h"
#include "pseudo_ops.h"
#include "condor_config.h"
#include "exit.h"
#include "filename_tools.h"
#include "basename.h"
#include "nullfile.h"
#include "spooled_job_files.h"
#include "condor_holdcodes.h"
#include "condor_event.h"

#include <filesystem>
#include "manifest.h"
#include "dc_coroutines.h"
#include "checkpoint_cleanup_utils.h"

#include "condor_base64.h"
#include "shortfile.h"
#include "single_provider_syndicate.h"

#include <regex>

#include "catalog_utils.h"
#include "job_ad_instance_recording.h"
#include "cxfer_state.h"
#include "classad_string_utils.h"


extern ReliSock *syscall_sock;
extern BaseShadow *Shadow;
extern RemoteResource *thisRemoteResource;
extern RemoteResource *parallelMasterResource;
extern CXFER_STATE cxfer_type;

static void append_buffer_info( std::string &url, const char *method, char const *path );
static int use_append( const char *method, const std::string &path );
static int use_compress( const char *method, const std::string &path );
static int use_fetch( const char *method, const std::string &path );
static int use_local_access( const std::string &file );

int
pseudo_register_machine_info(char * /* uiddomain */, char * /* fsdomain */, 
							 char * starterAddr, char *full_hostname )
{

	thisRemoteResource->setStarterAddress( starterAddr );
	thisRemoteResource->setMachineName( full_hostname );

		/* For backwards compatibility, if we get this old pseudo call
		   from the starter, we assume we're not going to get the
		   happy new pseudo_begin_execution call.  So, pretend we got
		   it now so we still log execute events and so on, even if
		   it's not as acurate as we'd like.
		*/
	thisRemoteResource->beginExecution();
	return 0;
}


int
pseudo_register_starter_info( ClassAd* ad )
{
	thisRemoteResource->setStarterInfo( ad );
	return 0;
}


int
pseudo_register_job_info(ClassAd* ad)
{
	// The starter sends an update with provisioned resources but no
	// JobState atrribute before the job begins execution.
	// Updates with a JobState attribute occur after the job is running.
	if (thisRemoteResource->getResourceState() == RR_STARTUP && ad->Lookup(ATTR_JOB_STATE)) {
		// We missed a begin_execution syscall, probably due to a disconnect.
		// Do the begin_execution logic now.
		dprintf(D_FULLDEBUG, "Received a register_job_info syscall without a begin_execution syscall. Doing begin_execution logic.\n");
		thisRemoteResource->beginExecution();
	}

	fix_update_ad(*ad);
	Shadow->updateFromStarterClassAd(ad);
	return 0;
}


int
pseudo_begin_execution()
{
	thisRemoteResource->beginExecution();
	return 0;
}


// In rare instances, the ad this function returns will need to be
// deleted by the caller. In those cases, delete_ad will be set to true.
// Otherwise, delete_ad will be set to false and the returned ad should
// not be deleted.
int
pseudo_get_job_info(ClassAd *&ad, bool &delete_ad)
{
	ClassAd * the_ad;
	delete_ad = false;

	the_ad = thisRemoteResource->getJobAd();
	ASSERT( the_ad );

	thisRemoteResource->initFileTransfer();

	Shadow->publishShadowAttrs( the_ad );

	ad = the_ad;

	return 0;
}


int
pseudo_get_user_info(ClassAd *&ad)
{
	static ClassAd* user_ad = NULL;


	if( ! user_ad ) {
			// if we don't have the ClassAd yet, allocate it and fill
			// it in with the info we care about
		user_ad = new ClassAd;

#ifndef WIN32
		user_ad->Assign( ATTR_UID, (int)get_user_uid() );

		user_ad->Assign( ATTR_GID, (int)get_user_gid() );
#endif

	}

	ad = user_ad;
	return 0;
}

// The list of attributes that some (old?) statrers try and incorrectly update
// this table is used to remove or rename them before we process the update ad.
//
typedef struct {
	const char * const updateAttr; // name in update ad
	const char * const newAttr;    // rename to this before processing the update ad, if NULL, delete the attribute
} AttrToAttr;
static const AttrToAttr updateAdBlacklist[] = {
	// Prior to 8.7.2, the starter incorrectly sends JobStartDate each time the job starts
	// but JobStartDate is defined to be the timestamp of the FIRST execution of the job
	// and it is set by the Schedd the first time it makes a shadow, so we want to just delete
	// this attribute.
	{ ATTR_JOB_START_DATE, NULL },
};

void fix_update_ad(ClassAd & update_ad)
{
	// remove or rename attributes in the update ad before we process it.
	for (size_t ii = 0; ii < COUNTOF(updateAdBlacklist); ++ii) {
		ExprTree * tree = update_ad.Remove(updateAdBlacklist[ii].updateAttr);
		if (tree) {
			if (IsDebugLevel(D_MACHINE)) {
				dprintf(D_MACHINE, "Update ad contained '%s=%s' %s it\n",
					updateAdBlacklist[ii].updateAttr, ExprTreeToString(tree),
					updateAdBlacklist[ii].newAttr ? "Renaming" : "Removing"
					);
			}
			if (updateAdBlacklist[ii].newAttr) {
				update_ad.Insert(updateAdBlacklist[ii].newAttr, tree);
			} else {
				delete tree;
			}
		}
	}
}

int
pseudo_job_exit(int status, int reason, ClassAd* ad)
{
	if (thisRemoteResource->getResourceState() == RR_STARTUP) {
		// We missed a begin_execution syscall, probably due to a disconnect.
		// Do the begin_execution logic now.
		dprintf(D_FULLDEBUG, "Received a job_exit syscall without a begin_execution syscall. Doing begin_execution logic.\n");
		thisRemoteResource->beginExecution();
	}

	// reset the reason if less than EXIT_CODE_OFFSET so that
	// an older starter can be made compatible with the newer
	// schedd exit reasons.
	if ( reason < EXIT_CODE_OFFSET ) {
		if ( reason != JOB_EXCEPTION && reason != DPRINTF_ERROR ) {
			reason += EXIT_CODE_OFFSET;
			dprintf(D_SYSCALLS, "in pseudo_job_exit: old starter, reason reset"
				" from %d to %d\n",reason-EXIT_CODE_OFFSET,reason);
		}
	}
	dprintf(D_SYSCALLS, "in pseudo_job_exit: status=%d,reason=%d\n",
			status, reason);

	// Despite what exit.h says, JOB_COREDUMPED is set by the starter only
	// when the job is NOT killed (and left a core file).
	if( reason == JOB_EXITED
		|| reason == JOB_EXITED_AND_CLAIM_CLOSING
		|| reason == JOB_COREDUMPED ) {
		thisRemoteResource->incrementJobCompletionCount();
	}
	fix_update_ad(*ad);
	thisRemoteResource->updateFromStarter( ad );
	thisRemoteResource->resourceExit( reason, status );
	Shadow->updateJobInQueue( U_STATUS );
	return 0;
}

int 
pseudo_job_termination( ClassAd *ad )
{
	bool exited_by_signal = false;
	bool core_dumped = false;
	int exit_signal = 0;
	int exit_code = 0;
	std::string exit_reason;

	if (thisRemoteResource->getResourceState() == RR_STARTUP) {
		// We missed a begin_execution syscall, probably due to a disconnect.
		// Do the begin_execution logic now.
		dprintf(D_FULLDEBUG, "Received a job_termination syscall without a begin_execution syscall. Doing begin_execution logic.\n");
		thisRemoteResource->beginExecution();
	}

	ad->LookupBool(ATTR_ON_EXIT_BY_SIGNAL,exited_by_signal);
	ad->LookupBool(ATTR_JOB_CORE_DUMPED,core_dumped);
	ad->LookupString(ATTR_EXIT_REASON,exit_reason);

	// Only one of these next two exist.
	ad->LookupInteger(ATTR_ON_EXIT_SIGNAL,exit_signal);
	ad->LookupInteger(ATTR_ON_EXIT_CODE,exit_code);

	// This will utilize only the correct arguments depending on if the
	// process exited with a signal or not.
	Shadow->mockTerminateJob( exit_reason, exited_by_signal, exit_code,
		exit_signal, core_dumped );

	return 0;
}


/*
If short_path is an absolute path, copy it to full path.
Otherwise, tack the current directory on to the front
of short_path, and copy it to full_path.
Notice that the old shadow kept track of the job as it
moved around, but there is no such notion in this shadow,
so CurrentWorkingDir is replaced with the job's iwd.
*/
 
static void complete_path( const char *short_path, std::string &full_path )
{
	if ( fullpath(short_path) ) {
		full_path = short_path;
	} else {
		formatstr(full_path, "%s%s%s",
						  Shadow->getIwd(),
						  DIR_DELIM_STRING,
						  short_path);
	}
}

/*
This call translates a logical path name specified by a user job
into an actual url which describes how and where to fetch
the file from.  For example, joe/data might become buffer:remote:/usr/joe/data
*/

int pseudo_get_file_info_new( const char *logical_name, char *&actual_url )
{
	std::string remap_list;
	std::string split_dir;
	std::string split_file;
	std::string full_path;
	std::string remap;
	std::string urlbuf;
	const char	*method = NULL;

	dprintf( D_SYSCALLS, "\tlogical_name = \"%s\"\n", logical_name );

	ASSERT( actual_url == NULL );

	/* The incoming logical name might be a simple, relative, or complete path */
	/* We need to examine both the full path and the simple name. */

	filename_split( logical_name, split_dir, split_file );
	complete_path( logical_name, full_path );

	/* Any name comparisons must check the logical name, the simple name, and the full path */

	if(Shadow->getJobAd()->LookupString(ATTR_FILE_REMAPS,remap_list) &&
	  (filename_remap_find( remap_list.c_str(), logical_name, remap ) ||
	   filename_remap_find( remap_list.c_str(), split_file.c_str(), remap ) ||
	   filename_remap_find( remap_list.c_str(), full_path.c_str(), remap ))) {

		dprintf(D_SYSCALLS,"\tremapped to: %s\n",remap.c_str());

		/* If the remap is a full URL, return right away */
		/* Otherwise, continue processing */

		if(strchr(remap.c_str(),':')) {
			dprintf(D_SYSCALLS,"\tremap is complete url\n");
			actual_url = strdup(remap.c_str());
			return 0;
		} else {
			dprintf(D_SYSCALLS,"\tremap is simple file\n");
			complete_path( remap.c_str(), full_path );
		}
	} else {
		dprintf(D_SYSCALLS,"\tnot remapped\n");
	}

	dprintf( D_SYSCALLS,"\tfull_path = \"%s\"\n", full_path.c_str() );

	/* Now, we have a full pathname. */
	/* Figure out what url modifiers to slap on it. */

	if( use_local_access(full_path) ) {
		method = "local";
	} else {
		method = "remote";
	}

	if( use_fetch(method,full_path) ) {
		urlbuf += "fetch:";
	}

	if( use_compress(method,full_path) ) {
		urlbuf += "compress:";
	}

	append_buffer_info(urlbuf,method,full_path.c_str());

	if( use_append(method,full_path) ) {
		urlbuf += "append:";
	}

	if (method) {
		urlbuf += method;
		urlbuf += ":";
	}
	urlbuf += full_path;
	actual_url = strdup(urlbuf.c_str());

	dprintf(D_SYSCALLS,"\tactual_url: %s\n",actual_url);

	return 0;
}

static void append_buffer_info( std::string &url, const char *method, char const *path )
{
	std::string buffer_list;
	std::string buffer_string;
	std::string dir;
	std::string file;
	int s,bs,ps;
	int result;

	filename_split(path,dir,file);

	/* Do not buffer special device files, whether local or remote */
	if(!strncmp(path,"/dev/",5)) return;

	/* Get the default buffer setting */
	pseudo_get_buffer_info( &s, &bs, &ps );

	/* Now check for individual file overrides */
	/* These lines have the same syntax as a remap list */

	if(Shadow->getJobAd()->LookupString(ATTR_BUFFER_FILES,buffer_list)) {
		if( filename_remap_find(buffer_list.c_str(),path,buffer_string) ||
			filename_remap_find(buffer_list.c_str(),file.c_str(),buffer_string) ) {

			/* If the file is merely mentioned, turn on the default buffer */
			url += "buffer:";

			/* If there is also a size setting, use that */
			result = sscanf(buffer_string.c_str(),"(%d,%d)",&s,&bs);
			if( result==2 ) url += buffer_string;

			return;
		}
	}

	/* Turn on buffering if the value is set and is not special or local */
	/* In this case, use the simple syntax 'buffer:' so as not to confuse old libs */

	if (s>0 && bs>0 && method && strcmp(method,"local")) {
		url += "buffer:";
	}
}

/* Return true if this JobAd attribute contains this path */

static int attr_list_has_file( const char *attr, const std::string &path )
{
	std::string str;

	const char *file = condor_basename(path.c_str());

	Shadow->getJobAd()->LookupString(attr,str);
	std::vector<std::string> list = split(str);

	if( contains_withwildcard(list, path) || contains_withwildcard(list, file) ) {
		return 1;
	} else {
		return 0;
	}
}

static int use_append( const char * /* method */, const std::string &path )
{
	return attr_list_has_file( ATTR_APPEND_FILES, path );
}

static int use_compress( const char * /* method */, const std::string &path )
{
	return attr_list_has_file( ATTR_COMPRESS_FILES, path );
}

static int use_fetch( const char * /* method */, const std::string &path )
{
	return attr_list_has_file( ATTR_FETCH_FILES, path );
}

/*
Return the buffer configuration.  If the classad contains nothing,
assume it is zero.
*/

int pseudo_get_buffer_info( int *bytes_out, int *block_size_out, int *prefetch_bytes_out )
{
	int bytes=0, block_size=0;

	Shadow->getJobAd()->LookupInteger(ATTR_BUFFER_SIZE,bytes);
	Shadow->getJobAd()->LookupInteger(ATTR_BUFFER_BLOCK_SIZE,block_size);

	if( bytes<0 ) bytes = 0;
	if( block_size<0 ) block_size = 0;
	if( bytes<block_size ) block_size = bytes;

	*bytes_out = bytes;
	*block_size_out = block_size;
	*prefetch_bytes_out = 0;

	dprintf(D_SYSCALLS,"\tbuffer configuration is bytes=%d block_size=%d\n",bytes, block_size );

	return 0;
}

static int use_local_access( const std::string &file )
{
	return
		file == "/dev/null" ||
		file == "/dev/zero" ||
		attr_list_has_file( ATTR_LOCAL_FILES, file );
}

int
pseudo_ulog( ClassAd *ad )
{
	// Ignore the event time we were given, use the
	// current time and timezone
	ad->Delete("EventTime");

	ULogEvent *event = instantiateEvent(ad);
	int result = 0;
	char const *critical_error = NULL;
	std::string CriticalErrorBuf;
	int hold_reason_code = 0;
	int hold_reason_sub_code = 0;

	if(!event) {
		std::string add_str;
		sPrintAd(add_str, *ad);
		dprintf(
		  D_ALWAYS,
		  "invalid event ClassAd in pseudo_ulog: %s\n",
		  add_str.c_str());
		return -1;
	}

	if( event->eventNumber == ULOG_REMOTE_ERROR ) {
		RemoteErrorEvent *err = (RemoteErrorEvent *)event;

		if(!err->getExecuteHost() || !*err->getExecuteHost()) {
			//Insert remote host information.
			char *execute_host = NULL;
			thisRemoteResource->getMachineName(execute_host);
			err->execute_host = execute_host;
			free(execute_host);
		}

		if(err->isCriticalError()) {
			formatstr(
			  CriticalErrorBuf,
			  "Error from %s: %s",
			  err->getExecuteHost(),
			  err->getErrorText());

			critical_error = CriticalErrorBuf.c_str();

			hold_reason_code = err->hold_reason_code;
			hold_reason_sub_code = err->hold_reason_subcode;
			if (hold_reason_code == 0) {
				hold_reason_code = CONDOR_HOLD_CODE::StarterError;
			}
		}
	}

	if( !Shadow->uLog.writeEvent( event, ad ) ) {
		std::string add_str;
		sPrintAd(add_str, *ad);
		dprintf(
		  D_ALWAYS,
		  "unable to log event in pseudo_ulog: %s\n",
		  add_str.c_str());
		result = -1;
	}

	if (critical_error) {
		// Let the RemoteResource know that the starter is shutting
		// down and failing to kill it it expected.
		thisRemoteResource->resourceExit( JOB_SHOULD_REQUEUE, -1 );
		Shadow->evictJob(JOB_SHOULD_REQUEUE, critical_error, hold_reason_code, hold_reason_sub_code);
	}

	delete event;
	return result;
}

int
pseudo_get_job_ad( ClassAd* &ad )
{
	RemoteResource *remote;
	if (parallelMasterResource == NULL) {
		remote = thisRemoteResource;
	} else {
		remote = parallelMasterResource;
	}
	ad = remote->getJobAd();
	return 0;
}


int
pseudo_get_job_attr( const char *name, std::string &expr )
{
	RemoteResource *remote;
	if (parallelMasterResource == NULL) {
		remote = thisRemoteResource;
	} else {
		remote = parallelMasterResource;
	}
	ClassAd *ad = remote->getJobAd();

	ExprTree *e = ad->LookupExpr(name);
	if(e) {
		expr = ExprTreeToString(e);
		dprintf(D_SYSCALLS,"pseudo_get_job_attr(%s) = %s\n",name,expr.c_str());
		return 0;
	} else {
		dprintf(D_SYSCALLS,"pseudo_get_job_attr(%s) is UNDEFINED\n",name);
		expr = "UNDEFINED";
		return 0;
	}
}

int
pseudo_set_job_attr( const char *name, const char *expr, bool log )
{
	RemoteResource *remote;
	if (parallelMasterResource == NULL) {
		remote = thisRemoteResource;
	} else {
		remote = parallelMasterResource;
	}
	if(Shadow->updateJobAttr(name,expr,log)) {
		dprintf(D_SYSCALLS,"pseudo_set_job_attr(%s,%s) succeeded\n",name,expr);
		ClassAd *ad = remote->getJobAd();
		ASSERT(ad);
		ad->AssignExpr(name,expr);
		return 0;
	} else {
		dprintf(D_SYSCALLS,"pseudo_set_job_attr(%s,%s) failed\n",name,expr);
		return -1;
	}
}

int
pseudo_constrain( const char *expr )
{
	std::string reqs;
	std::string newreqs;

	dprintf(D_SYSCALLS,"pseudo_constrain(%s)\n",expr);
	dprintf(D_SYSCALLS,"\tchanging AgentRequirements to %s\n",expr);

	if(pseudo_set_job_attr("AgentRequirements",expr)!=0) return -1;
	if(pseudo_get_job_attr("Requirements",reqs)!=0) return -1;

	if(strstr(reqs.c_str(),"AgentRequirements")) {
		dprintf(D_SYSCALLS,"\tRequirements already refers to AgentRequirements\n");
		return 0;
	} else {
		formatstr(newreqs, "(%s) && AgentRequirements", reqs.c_str());
		dprintf(D_SYSCALLS,"\tchanging Requirements to %s\n",newreqs.c_str());
		return pseudo_set_job_attr("Requirements",newreqs.c_str());
	}
}

int pseudo_get_sec_session_info(
	char const *starter_reconnect_session_info,
	std::string &reconnect_session_id,
	std::string &reconnect_session_info,
	std::string &reconnect_session_key,
	char const *starter_filetrans_session_info,
	std::string &filetrans_session_id,
	std::string &filetrans_session_info,
	std::string &filetrans_session_key)
{
	RemoteResource *remote;
	if (parallelMasterResource == NULL) {
		remote = thisRemoteResource;
	} else {
		remote = parallelMasterResource;
	}

	bool rc = remote->getSecSessionInfo(
		starter_reconnect_session_info,
		reconnect_session_id,
		reconnect_session_info,
		reconnect_session_key,
		starter_filetrans_session_info,
		filetrans_session_id,
		filetrans_session_info,
		filetrans_session_key);

	if( !rc ) {
		return -1;
	}
	return 1;
}


//
// Shadow-specific utility functions for dealing with checkpoint notifications.
//
void
deleteCheckpoint( const ClassAd * jobAd, int checkpointNumber ) {
	// Record on disk that this checkpoint attempt failed, so that
	// we won't worry about not being able to entirely delete it.
	std::string jobSpoolPath;
	SpooledJobFiles::getJobSpoolPath(jobAd, jobSpoolPath);
	std::filesystem::path spoolPath(jobSpoolPath);

	std::string manifestName;
	formatstr( manifestName, "_condor_checkpoint_MANIFEST.%.4d", checkpointNumber );
	std::string failureName;
	formatstr( failureName, "_condor_checkpoint_FAILURE.%.4d", checkpointNumber );
	std::error_code errorCode;
	std::filesystem::rename( spoolPath / manifestName, spoolPath / failureName, errorCode );
	if( errorCode.value() != 0 ) {
		// If no MANIFEST file was written, we can't clean up
		// anyway, so it doesn't matter if we didn't rename it.
		dprintf( D_FULLDEBUG,
			"Failed to rename %s to %s on checkpoint upload/validation failure, error code %d (%s)\n",
			(spoolPath / manifestName).string().c_str(),
			(spoolPath / failureName).string().c_str(),
			errorCode.value(), errorCode.message().c_str()
		);

		return;
	}

	// Clean up just this checkpoint attempt.  We don't actually
	// care if it succeeds; condor_preen is back-stopping us.
	//
	std::string checkpointDestination;
	if(! jobAd->LookupString( ATTR_JOB_CHECKPOINT_DESTINATION, checkpointDestination ) ) {
		dprintf( D_ALWAYS,
			"While handling a checkpoint event, could not find %s in job ad, which is required to attempt a checkpoint.\n",
			ATTR_JOB_CHECKPOINT_DESTINATION
		);

		return;
	}

	std::string globalJobID;
	jobAd->LookupString( ATTR_GLOBAL_JOB_ID, globalJobID );
	ASSERT(! globalJobID.empty());
	std::replace( globalJobID.begin(), globalJobID.end(), '#', '_' );

	formatstr( checkpointDestination, "%s/%s/%.4d",
		checkpointDestination.c_str(), globalJobID.c_str(),
		checkpointNumber
	);

	// The clean-up script is entitled to a copy of the job ad,
	// and the only way to give it one is via the filesystem.
	// It's clumsy, but for now, rather than deal with cleaning
	// this job ad file up, just store it in the job's SPOOL.
	// (Jobs with a checkpoint destination set don't transfer
	// anything from SPOOL except for the MANIFEST files, so
	// this won't cause problems even if the .job.ad file is
	// written by the starter before file transfer rather than
	// after.)
	std::string jobAdPath = jobSpoolPath;

	// FIXME: This invocation assumes that it's OK to block here
	// in the shadow until the clean-up attempt is done.  We'll
	// probably need to (refactor it into the cleanup utils and)
	// call spawnCheckpointCleanupProcessWithTimeout() -- or
	// something very similar -- and plumb through an additional
	// option specifying which specific checkpoint to clean-up.
	std::string error;
	manifest::deleteFilesStoredAt( checkpointDestination,
		(spoolPath / failureName).string(),
		jobAdPath,
		error,
		true /* this was a failed checkpoint */
	);

	// It's OK that we don't remove the .job.ad file and the
	// corresponding parent directory after a successful clean-up;
	// we know we'll need them again later, since we aren't
	// deleting all of the job's checkpoints, and the schedd's
	// call to condor_manifest will delete them when job exits
	// the queue.
}


//
// This syscall MUST ignore information it doesn't know how to deal with.
//
// The thinking for this syscall is the following: as Miron asks for more
// metrics, it's highly likely that there will be other event notifications
// that the starter needs to send to the shadow.  Rather than create a
// semantically significant remote syscall for each one, let's just create
// a single one that they can all use.  (As a new syscall, pools won't see
// correct values for the new metrics until both the shadow and the starter
// have been upgraded.)  However, rather than writing a bunch of complicated
// code to determine which version(s) can send what events to which
// version(s), we can just declare and require that this function just
// ignore attributes it doesn't know how to deal with (and not fail if an
// attribute has the wrong type).
//

int
pseudo_event_notification( const ClassAd & ad ) {
	std::string eventType;
	if(! ad.LookupString( "EventType", eventType )) {
		return GENERIC_EVENT_RV_NO_ETYPE;
	}

	ClassAd * jobAd = Shadow->getJobAd();
	ASSERT(jobAd);

	if( eventType == "ActivationExecutionExit" ) {
		thisRemoteResource->recordActivationExitExecutionTime(time(NULL));
	} else if( eventType == "SuccessfulCheckpoint" ) {
		std::string checkpointDestination;
		if(! jobAd->LookupString( ATTR_JOB_CHECKPOINT_DESTINATION, checkpointDestination ) ) {
			dprintf( D_TEST, "Not attempting to clean up checkpoints going to SPOOL.\n" );
			return GENERIC_EVENT_RV_OK;
		}

		int checkpointNumber = -1;
		if( ad.LookupInteger( ATTR_JOB_CHECKPOINT_NUMBER, checkpointNumber ) ) {
			unsigned long numToKeep = 1 + param_integer( "DEFAULT_NUM_EXTRA_CHECKPOINTS", 1 );
			dprintf( D_STATUS | D_VERBOSE, "Checkpoint number %d succeeded, deleting all but the most recent %lu successful checkpoints.\n", checkpointNumber, numToKeep );

			// Before we can delete a checkpoint, it needs to be moved from the
			// job's spool to the job owner's checkpoint-cleanup directory.  So
			// we need to figure out which checkpoint(s) we want to keep and
			// move all of the other ones.

			std::string spoolPath;
			SpooledJobFiles::getJobSpoolPath( jobAd, spoolPath );
			std::filesystem::path spool( spoolPath );
			if(! (std::filesystem::exists(spool) && std::filesystem::is_directory(spool))) {
				dprintf(D_STATUS, "Checkpoint suceeded but job spool directory either doesn't exist or isn't a directory; not trying to clean up old checkpoints.\n" );
				return GENERIC_EVENT_RV_OK;
			}

			std::error_code errCode;
			std::set<long> checkpointsToSave;
			auto spoolDir = std::filesystem::directory_iterator(spool, errCode);
			if( errCode ) {
				dprintf( D_STATUS, "Checkpoint suceeded but job spool directory couldn't be checked for old checkpoints, not trying to clean them up: %d '%s'.\n", errCode.value(), errCode.message().c_str() );
				return GENERIC_EVENT_RV_OK;
			}
			for( const auto & entry : spoolDir ) {
				const auto & stem = entry.path().stem();
				if( starts_with(stem.string(), "_condor_checkpoint_") ) {
					bool success = ends_with(stem.string(), "MANIFEST");
					bool failure = ends_with(stem.string(), "FAILURE");
					if( success || failure ) {
						char * endptr = NULL;
						const auto & suffix = entry.path().extension().string().substr(1);
						long manifestNumber = strtol( suffix.c_str(), & endptr, 10 );
						if( endptr == suffix.c_str() || *endptr != '\0' ) {
							dprintf( D_FULLDEBUG, "Unable to extract checkpoint number from '%s', skipping.\n", entry.path().string().c_str() );
							continue;
						}

						if( success ) {
							checkpointsToSave.insert(manifestNumber);
							if( checkpointsToSave.size() > numToKeep ) {
								checkpointsToSave.erase(checkpointsToSave.begin());
							}
						}
					}
				}
			}

			std::string buffer;
			formatstr( buffer, "Last %lu succesful checkpoints were: ", numToKeep );
			for( const auto & value : checkpointsToSave ) {
				formatstr_cat( buffer, "%ld ", value );
			}
			dprintf( D_STATUS | D_VERBOSE, "%s\n", buffer.c_str() );

			// Move all but checkpointsToSave's MANIFEST/FAILURE files.  The
			// schedd won't interrupt because it never does anything to the
			// job's spool while the shadow is running, and we won't be moving
			// (or removing) the shared intermediate directories: the schedd
			// will clean those up when the job leaves the queue, as normal.

			int cluster, proc;
			jobAd->LookupInteger( ATTR_CLUSTER_ID, cluster );
			jobAd->LookupInteger( ATTR_PROC_ID, proc );
			if(! moveCheckpointsToCleanupDirectory(
				cluster, proc, jobAd, checkpointsToSave
			)) {
				// Then there's nothing we can do here.
				return GENERIC_EVENT_RV_OK;
			}

			int CLEANUP_TIMEOUT = param_integer( "SHADOW_CHECKPOINT_CLEANUP_TIMEOUT", 300 );
			spawnCheckpointCleanupProcessWithTimeout( cluster, proc, jobAd, CLEANUP_TIMEOUT );
		}
	} else if( eventType == "FailedCheckpoint" ) {
		int checkpointNumber = -1;
		if( ad.LookupInteger( ATTR_JOB_CHECKPOINT_NUMBER, checkpointNumber ) ) {
			dprintf( D_STATUS, "Checkpoint number %d failed, deleting it and updating schedd.\n", checkpointNumber );

			deleteCheckpoint( jobAd, checkpointNumber );

			// If the checkpoint upload succeeded, we have an on-disk (in
			// SPOOL) record of its success, and we'll find whether or not the
			// next iteration of the starter has the most-recent checkpoint
			// number.  Howver, if the checkpoint upload failed hard enough,
			// we may not have a failure manifest, so we should make sure the
			// next iteration of the starter doesn't attempt to overwrite an
			// existing checkpoint (and fail as a result).
			jobAd->Assign( ATTR_JOB_CHECKPOINT_NUMBER, checkpointNumber );
			Shadow->updateJobInQueue( U_PERIODIC );
		}
	} else if( eventType == "InvalidCheckpointDownload" ) {
		int checkpointNumber = -1;
		if(! ad.LookupInteger( ATTR_JOB_CHECKPOINT_NUMBER, checkpointNumber )) {
			dprintf( D_ALWAYS, "Starter sent an InvalidCheckpointDownload event notification, but the job has no checkpoint number; ignoring.\n" );
			return GENERIC_EVENT_RV_INCOMPLETE;
		}

		//
		// The starter can't just restart the job from scratch, because
		// the sandbox may have been modified by the attempt to transfer
		// the checkpoint -- we don't require that our plug-ins are
		// all-or-nothing.
		//
		dprintf( D_STATUS, "Checkpoint number %d was invalid, rolling back.\n", checkpointNumber );

		//
		// We should absolutely write an event to job's event log here
		// noting that the checkpoint was invalid and that we're rolling
		// back, but I'd have to think about what that event should
		// look like.
		//

		//
		// This moves the MANIFEST file out of the way, so that  even if
		// the deletion proper fails, we won't try to use the checkpoint
		// again.
		//
		// Arguably, we should distinguish between storage and retrieval
		// failures when we move the MANIFEST file, but I think it makes
		// sense to accept not being able to fully-delete a corrupt checkpoint.
		//
		deleteCheckpoint( jobAd, checkpointNumber );

		// Do NOT set the checkpoint number.  Either the most-recent
		// checkpoint failed to validate, in which case it's redundant,
		// or an earlier one did, and we don't want to cause an
		// upload failure later by overwriting it.  (Checkpoint numbers
		// should be monotonically increasing.)

		//
		// If it becomes necessary, the shadow could go into its
		// exit-to-requeue routine here.
		//
		return GENERIC_EVENT_RV_OK;
	} else if( eventType == ETYPE_DIAGNOSTIC_RESULT ) {
		dprintf( D_ALWAYS, "Received diagnostic result.\n" );

		std::string diagnostic;
		if(! ad.LookupString( ATTR_DIAGNOSTIC, diagnostic )) {
			dprintf( D_ALWAYS, "Starter sent a diagnostic result, but did not name which diagnostic; ignoring.\n" );
			return GENERIC_EVENT_RV_INCOMPLETE;
		}

		std::string result;
		if(! ad.LookupString( "Result", result ) ) {
			dprintf( D_ALWAYS, "Starter sent a diagnostic result for '%s', but it had no result.\n", diagnostic.c_str() );
			return GENERIC_EVENT_RV_INCOMPLETE;
		}

		if( result != "Completed" ) {
			dprintf( D_ALWAYS, "Diagnostic '%s' did not complete: '%s'\n",
				diagnostic.c_str(), result.c_str()
			);
			return GENERIC_EVENT_RV_OK;
		}


		int exitStatus;
		if(! ad.LookupInteger( "ExitStatus", exitStatus ) ) {
			dprintf( D_ALWAYS, "Starter sent a completed diagnostic result for '%s', but it had no exit status.\n", diagnostic.c_str() );
			return GENERIC_EVENT_RV_INCOMPLETE;
		}

		std::string contents;
		if(! ad.LookupString( "Contents", contents ) ) {
			dprintf( D_ALWAYS, "Starter sent a completed diagnostic result for '%s', but it had no contents.\n", diagnostic.c_str() );
			return GENERIC_EVENT_RV_INCOMPLETE;
		}

		int decoded_bytes = 0;
		unsigned char * decoded = nullptr;
		condor_base64_decode( contents.c_str(), & decoded, & decoded_bytes, false );
		if( decoded == NULL ) {
			dprintf( D_ALWAYS, "Failed to decode contents of diagnostic result for '%s'.\n", diagnostic.c_str() );
			return GENERIC_EVENT_RV_INVALID;
		}
		decoded[decoded_bytes] = '\0';

		if( exitStatus != 0 ) {
			dprintf( D_ALWAYS, "Starter sent a completed diagnostic result for '%s', but its exit status was non-zero (%d)\n", diagnostic.c_str(), exitStatus );
			dprintf( D_FULLDEBUG, "Output to first NUL follows: '%s'\n", decoded );
			free(decoded);
			return GENERIC_EVENT_RV_OK;
		}

		if( diagnostic != DIAGNOSTIC_SEND_EP_LOGS ) {
			dprintf( D_ALWAYS, "Starter sent an unexpected diagnostic result (for '%s'); ignoring.\n", diagnostic.c_str() );
			free(decoded);
			return GENERIC_EVENT_RV_CONFUSED;
		}

#ifdef TODDT_SAID_OTHERWISE
		// Write `decoded` to a well-known location.  We should probably
		// add a job-ad attribute which controls the location.
		std::string jobIWD;
		if( jobAd->LookupString( ATTR_JOB_IWD, jobIWD ) ) {
			std::filesystem::path iwd(jobIWD);
			std::filesystem::path diagnostic_dir( iwd / ".diagnostic" );

			if(! std::filesystem::exists( diagnostic_dir )) {
				std::error_code ec;
				std::filesystem::create_directory( diagnostic_dir, ec );
			}

			// I'm thinking that any logging here would be uninteresting.
			long long int clusterID = 0, procID = 0, numJobStarts = 0;
			jobAd->LookupInteger( ATTR_CLUSTER_ID, clusterID );
			jobAd->LookupInteger( ATTR_PROC_ID, procID );
			jobAd->LookupInteger( ATTR_NUM_JOB_STARTS, numJobStarts );

			std::string name;
			formatstr( name, "%s.%lld.%lld.%lld", diagnostic.c_str(), clusterID, procID, numJobStarts );
			std::filesystem::path output( diagnostic_dir / name );
			if(! htcondor::writeShortFile( output.string(), decoded, decoded_bytes )) {
				dprintf( D_ALWAYS, "Failed to write output for diagnostic '%s' to %s\n", diagnostic.c_str(), output.string().c_str() );
			}
		} else {
			dprintf( D_ALWAYS, "No IWD in job ad, not writing output for diagnostic '%s'\n", diagnostic.c_str() );
		}
#else

		// Write 'decoded' to the log, one prefixed line at a time.
		for( const auto & line : StringTokenIterator( (char *)decoded, "\n" ) ) {
			dprintf( D_ALWAYS, "[diagnostic log] >>> %s\n", line.c_str() );
		}

#endif

		free( decoded );
		return GENERIC_EVENT_RV_OK;
	} else {
		dprintf( D_ALWAYS, "Ignoring unknown event type '%s'\n", eventType.c_str() );
	}

	return GENERIC_EVENT_RV_UNKNOWN;
}


bool
LookupAdInContext( const ClassAd & ad, const std::string attr, ClassAd * & value ) {
	auto * ctx = ad.Lookup( ATTR_CONTEXT_AD );
	const ClassAd * context = dynamic_cast<ClassAd *>(ctx);
	if(! context) {
		return false;
	}

	auto * cad = context->Lookup( attr );
	ClassAd * classAd = dynamic_cast<ClassAd *>(cad);
	if( classAd ) {
		value = classAd;
		return true;
	}
	return false;
}


bool
LookupIntInContext( const ClassAd & ad, const std::string & attr, int & value ) {
	auto * ctx = ad.Lookup( ATTR_CONTEXT_AD );
	const ClassAd * context = dynamic_cast<ClassAd *>(ctx);
	if( context ) {
		return context->LookupInteger( attr, value );
	}
	return false;
}

bool
LookupIntInContext( const ClassAd & ad, const std::string & attr, long long & value ) {
	auto * ctx = ad.Lookup( ATTR_CONTEXT_AD );
	const ClassAd * context = dynamic_cast<ClassAd *>(ctx);
	if( context ) {
		return context->LookupInteger( attr, value );
	}
	return false;
}


bool
LookupBoolInContext( const ClassAd & ad, const std::string & attr, bool & value ) {
	auto * ctx = ad.Lookup( ATTR_CONTEXT_AD );
	const ClassAd * context = dynamic_cast<ClassAd *>(ctx);
	if( context ) {
		return context->LookupBool( attr, value );
	}
	return false;
}


bool
LookupStringInContext( const ClassAd & ad, const std::string & attr, std::string & value ) {
	ClassAd * context = dynamic_cast<ClassAd *>(ad.Lookup( ATTR_CONTEXT_AD ));
	if( context ) {
		return context->LookupString( attr, value );
	}
	return false;
}


//
// The ClassAds are all deliberately copies, because it's way easier to be
// marginally less efficient than to worry about heap object lifetimes
// (especially considering that locals from the caller won't be preserved
// across the co_yield() statements).
//
condor::cr::Piperator<ClassAd, ClassAd>
start_input_transfer_failure_conversation( ClassAd request ) {
	ClassAd guidance;

	// Step one: gather logs.
	guidance.InsertAttr(ATTR_COMMAND, COMMAND_RUN_DIAGNOSTIC);
	guidance.InsertAttr(ATTR_DIAGNOSTIC, DIAGNOSTIC_SEND_EP_LOGS);
	request = co_yield guidance;

	// Step two: abort the job.
	guidance.Clear();
	guidance.InsertAttr(ATTR_COMMAND, COMMAND_ABORT);
	co_return guidance;
}


ClassAd
UniShadow::before_common_file_transfer(
	const std::string & cifName,
	const std::string & commonInputFiles
) {
	dprintf( D_ALWAYS, "Starting common files transfer.\n" );
	CommonFilesEvent cfStartEvent;
	cfStartEvent.setType( CommonFilesEventType::TransferStarted );
	uLog.writeEvent( & cfStartEvent, jobAd );


	ClassAd commonAd;
	CopyAttribute(
		ATTR_JOB_IWD, commonAd,
		ATTR_JOB_IWD, * Shadow->getJobAd()
	);
	commonAd.InsertAttr( ATTR_TRANSFER_INPUT_FILES, commonInputFiles );
	// If we neglect to set ATTR_TRANSFER_EXECUTABLE, the FTO adds the
	// job's (non-existent) spool directory to the input list!
	commonAd.InsertAttr( ATTR_TRANSFER_EXECUTABLE, false );


	this->commonFTO = new FileTransfer();
	if( this->commonFTO == NULL ) {
		// This is bad, but it's consistent with failing to Init().
		EXCEPT( "UniShadow::do_common_file_transfer(): new FileTransfer() failed." );
	}
	// This sets ATTR_TRANSFER_[SOCKET|KEY] in `commonAd` "for us."
	int rval = this->commonFTO->Init( & commonAd, false, PRIV_USER, false );
	if( rval == 0 ) {
		// This is bad, but it's consistent with what we do in RemoteResouce::initFileTransfer().
		EXCEPT( "UniShadow::do_common_file_transfer(): Init() failed." );
	}
	this->commonFTO->setPeerVersion( this->getStarterVersion() );
	// This is broken on Windows, but we really want our timer to fire
	// during the transfer process.  Not clear if the progress function
	// will always be called often enough to be useful for keeping-alive.
	this->commonFTO->SetServerShouldBlock( false );


	ClassAd guidance;
	CopyAttribute(
		ATTR_TRANSFER_SOCKET, guidance,
		ATTR_TRANSFER_SOCKET, commonAd
	);
	CopyAttribute(
		ATTR_TRANSFER_KEY, guidance,
		ATTR_TRANSFER_KEY, commonAd
	);

	guidance.InsertAttr( ATTR_NAME, cifName );
	guidance.InsertAttr( ATTR_COMMAND, COMMAND_STAGE_COMMON_FILES );
	guidance.InsertAttr( ATTR_COMMON_INPUT_FILES, commonInputFiles );
	return guidance;
}


bool
UniShadow::after_common_file_transfer(
    const ClassAd & request,
    const std::string & cifName,
    std::string & stagingDir,
    long long & size
) {
	bool success = false;
	LookupBoolInContext( request, ATTR_RESULT, success );


	// This produces a second record for non-CEDAR transfers, which
	// is sub-optimal.  We can't, in general (because of simultaneous
	// transfers) avoid having up to one COMMON record per common files
	// group, but we should avoid having two.
	//
	// The plugin results are transferred before file transfer completes
	// (from the point of view of code that isn't in the FTO itself),
	// so only write an entry if here if CEDAR is the _only_ protocol.
	const ClassAd & statsAd = this->commonFTO->GetInfo().stats;
	std::string protocols;
	if( statsAd.LookupString("Protocols", protocols) ) {
		if( 0 == strcasecmp( protocols.c_str(), "cedar" ) ) {
			writeAdWithContextToEpoch( & statsAd, jobAd, "COMMON" );
		}
	}

	CommonFilesEvent cfFinishEvent;
	cfFinishEvent.setType( CommonFilesEventType::TransferFinished );
	uLog.writeEvent( & cfFinishEvent, jobAd );
	dprintf( D_ALWAYS, "Finished common files transfer: %s.\n", success ? "success" : "failure" );

	SingleProviderSyndicate * cfLock = this->cfLocks[cifName];
	// ASSERT(cfLock != NULL);

	if( success ) {
		// We'll assume that malformed replies are transients.
		if(! LookupStringInContext( request, ATTR_STAGING_DIR, stagingDir )) {
			dprintf( D_ALWAYS, "UniShadow::start_common_input_conversation(): malformed reply to doing common files transfer; aborting job.\n" );

			// We can't just release the cfLock, because that won't do
			// anything if another shadow is already waiting on us.
			if( cfLock != NULL ) {
				this->cfLocks.erase(cifName);
				delete cfLock;
			}

			// Consider replacing this with a call to evictJob().
			this->jobAd->Assign(ATTR_LAST_VACATE_TIME, time(nullptr));
			this->jobAd->Assign(ATTR_VACATE_REASON, "Starter sent malformed reply when asked to stage common files." );
			this->jobAd->Assign(ATTR_VACATE_REASON_CODE, CONDOR_HOLD_CODE::JobNotStarted);
			this->jobAd->Assign(ATTR_VACATE_REASON_SUBCODE, JOB_NOT_STARTED_SUB_CODE::CommonTransferBadReply);
			remRes->setExitReason(JOB_CXFER_FAILED_REQUEUE);
			remRes->killStarter(false);

			return false;
		}

		dprintf( D_TEST, "Staging successful, calling ready(%s)\n", stagingDir.c_str() );
		if(cfLock != NULL && ! cfLock->ready( stagingDir )) {
			// We failed to tell the jobs waiting on us that we're
			// ready.  This job can continue, but the others can't.
			//
			// We can't just call release(), because that doesn't
			// do anything if other jobs are waiting.  Deleting
			// the keyfile, however, will select a new lockholder.
			this->cfLocks.erase(cifName);
			delete cfLock;
		}

		// Starters before 25.12 don't send this.
		LookupIntInContext( request, ATTR_SIZE, size );
	} else {
		dprintf( D_ALWAYS, "UniShadow::start_common_input_conversation(): common file transfer failed, aborting job.\n" );

		// See previous commentary.
		SingleProviderSyndicate * cfLock = this->cfLocks[cifName];
		// ASSERT(cfLock != NULL);
		if( cfLock != NULL ) {
			this->cfLocks.erase(cifName);
			delete cfLock;
		}


		// Determine if the _request_ failed.  (If it succeeded, then we
		// can check to see if the _transfer_ failed.)  If it failed,
		// reschedule the job.
		bool should_cool_down = false;

		int requestResultCode = (int)RequestResult::Invalid;
		LookupIntInContext( request, ATTR_REQUEST_RESULT_CODE, requestResultCode );
		if( requestResultCode != (int)RequestResult::Invalid ) {
			should_cool_down = true;
			this->jobAd->Assign( ATTR_REQUEST_RESULT_CODE, requestResultCode );
		}

		int requestResultSubcode = (int)RequestResult::Invalid;
		LookupIntInContext( request, ATTR_REQUEST_RESULT_SUBCODE, requestResultSubcode );
		if( requestResultSubcode != (int)RequestResult::Invalid ) {
			should_cool_down = true;
			this->jobAd->Assign( ATTR_REQUEST_RESULT_SUBCODE, requestResultSubcode );
		}

		// According ToddT (on 2025-08-21), the `try_again` field
		// has only ever been intended for immediate starter-side
		// retries of CEDAR transfers.  (There's certainly nothing
		// in the plug-in interface, ignoring the incomplete
		// HTCONDOR-3064.)  So we'll just ignore it here; that means
		// that the only case we vacate for is a failure to attempt
		// common files transfer at all.
		if( should_cool_down ) {
			// Consider replacing this with a delayed (zero-second
			// timer) call to evictJob().
			this->jobAd->Assign(ATTR_LAST_VACATE_TIME, time(nullptr));
			this->jobAd->Assign(ATTR_VACATE_REASON, "Common file transfer failed to start.");
			// See comment below about which codes we're using here.
			this->jobAd->Assign(ATTR_VACATE_REASON_CODE, CONDOR_HOLD_CODE::JobNotStarted);
			this->jobAd->Assign(ATTR_VACATE_REASON_SUBCODE, JOB_NOT_STARTED_SUB_CODE::CommonTransferFailed);
			remRes->setExitReason(JOB_CXFER_FAILED_REQUEUE);
			remRes->killStarter(false);
		}


		//
		// Improve the starter's hold/vacate reason.
		//
		// From JICShadow::transferInputStatus().
		//
		const FileTransfer::FileTransferInfo info = this->commonFTO->GetInfo();

		std::string reason = "Failed to transfer files: ";
		if(! info.error_desc.empty()) {
			reason += info.error_desc;
		} else {
		    reason += " reason unknown.";
		}

		// This is not common file -specific...
		// we know if it's a hold or a vacate.
		int code = info.hold_code;
		int subcode = info.hold_subcode;
		std::string url_file_type;
		std::string improved_reason = improveReasonAttributes(
				reason.c_str(),
				code, subcode, url_file_type
		);

		// ... so go ahead and clean it up now.
		if( improved_reason.empty() ) {
			improved_reason = reason;
		} else {
			std::regex r("Transfer input files failure");
			improved_reason = std::regex_replace(
				improved_reason, r,
				"Transfer common input files failure"
			);
		}

		// FIXME: Will this be correct for a starter-side failure?
		dprintf( D_ZKM, "Shadow-side hold reason, code, and subcode: %s, %d, %d\n",
			info.error_desc.c_str(), info.hold_code, info.hold_subcode
		);

		//
		// There's no corresponding job, so exiting with JOB_SHOULD_HOLD
		// is meaningless for transfer shadows; however, the current
		// schedd code understands this and reacts as best it can.  In
		// the future, we should exit with JOB_CXFER_FAILED_REQUEUE or
		// -- after HTCONDOR-3379 -- with with JOB_CXFER_FAILED_HOLD.
		//

		// This seems wrong -- improveReasonAttributes() could
		// have changed the hold code and sub-code even if it
		// didn't produce and improved string -- but it's how
		// the original code in evictJob() worked.
		if( improved_reason.empty() ) {
			holdJob( info.error_desc.c_str(), code, subcode );
		} else {
			// improveReasonAttribute() will not specify that
			// job is going on hold because of _common_ input
			// transfer, so fix it here.
			std::regex r("Transfer input files failure");
			improved_reason = std::regex_replace(
				improved_reason, r,
				"Transfer common input files failure"
			);
			holdJob( improved_reason.c_str(), code, subcode );
		}

		return false;
	}

	return true;
}


ClassAd
do_wiring_up( const std::string & stagingDir, const std::string & cifName ) {
	ClassAd guidance;

	guidance.InsertAttr( ATTR_NAME, cifName );
	guidance.InsertAttr( ATTR_COMMAND, COMMAND_MAP_COMMON_FILES );
	guidance.InsertAttr( ATTR_STAGING_DIR, stagingDir );

	return guidance;
}


ClassAd
UniShadow::handle_wiring_failure() {
	// Consider replacing this with a (zero-second timer) call to evictJob().
	this->jobAd->Assign(ATTR_LAST_VACATE_TIME, time(nullptr));
	this->jobAd->Assign(ATTR_VACATE_REASON, "Failed to map files." );
	this->jobAd->Assign(ATTR_VACATE_REASON_CODE, CONDOR_HOLD_CODE::JobNotStarted);
	this->jobAd->Assign(ATTR_VACATE_REASON_SUBCODE, JOB_NOT_STARTED_SUB_CODE::CommonMappingFailed);
	remRes->setExitReason(JOB_MAPPING_FAILED);
	remRes->killStarter(false);

	ClassAd guidance;
	guidance.InsertAttr(ATTR_COMMAND, COMMAND_ABORT);
	return guidance;
}


template<> std::string
join<ListOfCatalogs>(const ListOfCatalogs & list, const char* delim) {
	std::string str;
	if (!list.empty()) {
		auto it = list.begin();
		str = it->first; ++it;
		for( ; it != list.end(); ++it ) {
			str += delim;
			str += it->first;
		}
	}
	return str;
}


//
// Rather than maintain an array of timers, one for each provisioned
// catalog, maintain an array of provisioned catalogs iterated over
// by a single timer.
//

void
UniShadow::set_provider_keep_alive( const std::string & cifName ) {
	static std::vector<std::string> cifNames;
	cifNames.push_back(cifName);

	if( this->producer_keep_alive == -1 ) {
		this->producer_keep_alive = daemonCore->Register_Timer(
			0, 60,
			[this] (int /* timerID */) -> void {
				// If this timer happens to fire on our way out the door, don't asplode.
				for( const auto & cifName : cifNames ) {
					if( this->cfLocks.contains(cifName) ) {
						SingleProviderSyndicate * cfLock = this->cfLocks[cifName];
						if( cfLock == NULL ) { continue; }

						// dprintf( D_ZKM, "Elected producer touch()ing keyfile.\n" );
						if(! cfLock->touch()) {
							// I boldly claim that the global destructor will delete
							// the corresponding cfLock.
							EXCEPT( "Elected producer touch() failed, aborting.\n" );
						}
					}
				}
			},
			"SingleProviderSyndicate producer keep-alive"
		);
		if( this->producer_keep_alive == -1 ) {
			// I boldly claim that the global destructor will delete the corresponding cfLock.
			EXCEPT( "Elected producer couldn't register keep-alive, aborting.\n" );
		}
	}
}


ClassAd
UniShadow::vacate_requeue_abort(
	const std::string & holdMessage, CONDOR_HOLD_CODE holdCode, int holdSubCode,
	const char * file, int line
) {
	dprintf( D_ALWAYS, "%s:%d: %s\n", file, line, holdMessage.c_str() );

	// Consider replacing this with a call to evictJob().
	this->jobAd->Assign(ATTR_LAST_VACATE_TIME, time(nullptr));
	this->jobAd->Assign(ATTR_VACATE_REASON, holdMessage );
	this->jobAd->Assign(ATTR_VACATE_REASON_CODE, holdCode);
	this->jobAd->Assign(ATTR_VACATE_REASON_SUBCODE, holdSubCode);
	remRes->setExitReason(JOB_CXFER_FAILED_REQUEUE);
	remRes->killStarter(false);


	ClassAd guidance;
	guidance.InsertAttr(ATTR_COMMAND, COMMAND_ABORT);
	return guidance;
}


#define VACATE_REQUEUE_ABORT(x,y,z) vacate_requeue_abort(x, y, z, __FILE__, __LINE__)


// The parameters are copies to simplify thinking about this coroutine.
condor::cr::Piperator<ClassAd, ClassAd>
UniShadow::start_staging_only_conversation(
	ClassAd request,
	ListOfCatalogs common_file_catalogs,
	std::map<std::string, std::string> internalToSimpleNameMap
) {
	bool success;

	// Transfer each catalog.  The caller has filtered the list in the
	// job ad, if appropriate.  (Since we never need to map the catalogs
	// in a transfer shadow, we don't need to know which ones weren't
	// transferred.)

	std::map<std::string, long long> cifNameToSizeMap;
	std::map<std::string, std::string> cifNameToStagingDirMap;
	for( const auto & [cifName, commonInputFiles] : common_file_catalogs ) {
		dprintf( D_ZKM, "%s = %s\n", cifName.c_str(), commonInputFiles.c_str() );

		long long size = -1;
		std::string stagingDir;
		ClassAd guidance = before_common_file_transfer( cifName, commonInputFiles );
		request = co_yield guidance;
		success = after_common_file_transfer( request, cifName, stagingDir, size );
		if(! success) {
			guidance.Clear();
			guidance.InsertAttr(ATTR_COMMAND, COMMAND_ABORT);
			co_return guidance;
		}

		cifNameToStagingDirMap[cifName] = stagingDir;
		cifNameToSizeMap[cifName] = size;
	}

	//
	// Although the starter could color (and recolor) the slot after each
	// common files transfer, it seems a little more flexible to let the
	// shadow decide when it happens and what the color is.  This also
	// lets us side-step adding global state to the starter tracking which
	// coloring has already been done (because the startd command to color
	// a slot replaces the previous color).
	//
	ClassAd guidance;
	guidance.InsertAttr( ATTR_COMMAND, COMMAND_COLOR_SLOT );


	//
	// Only starter versions >= 25.12 can handle version 2 commands.
	//
	int version_to_send = 1;
	CondorVersionInfo cvi( remRes->starter_version.c_str() );
	if( cvi.built_since_version( 25, 12, 0 ) ) {
		version_to_send = 2;
	}
	ClassAd * commonCatalogsAd = new ClassAd();
	commonCatalogsAd->InsertAttr( ATTR_VERSION, version_to_send );

	std::vector<ExprTree *> catalogAds;
	for( const auto & [cifName, stagingDir] : cifNameToStagingDirMap ) {
		ClassAd * catalogAd = new ClassAd();

		if( version_to_send == 1 ) {
			catalogAd->InsertAttr( ATTR_NAME, cifName );
			catalogAd->InsertAttr( "StagingDir", stagingDir );
			catalogAd->InsertAttr( "SimpleName", internalToSimpleNameMap[cifName] );
		} else {
			catalogAd->InsertAttr( ATTR_CATALOG, internalToSimpleNameMap[cifName] );
			catalogAd->InsertAttr( ATTR_CATALOG_PATH, stagingDir );
			catalogAd->InsertAttr( ATTR_CATALOG_SIZE, cifNameToSizeMap[cifName] );

			auto r = determineCIFScopeAndType( * jobAd );
			if(! r) { EXCEPT("Failed to determine CIF scope and type after having staged it."); }
			auto [scope, type] = * r;
			catalogAd->InsertAttr( ATTR_CATALOG_SCOPE, scope );
			catalogAd->InsertAttr( ATTR_CATALOG_SCOPE_TYPE, type );

			std::string access_point;
			jobAd->LookupString( ATTR_GLOBAL_JOB_ID, access_point );
			access_point = split(access_point, "#")[0];
			catalogAd->InsertAttr( ATTR_ACCESS_POINT, access_point );

			// This is kind of a hack, since the schedd already computed
			// the catalog ID, but I'm not sure we want the simple-to-ID
			// map in every job ad.
			auto c = computeCatalogID( * jobAd, internalToSimpleNameMap[cifName] );
			if(! c) { EXCEPT("Failed to compute catalog ID."); }
			catalogAd->InsertAttr( ATTR_CATALOG_ID, * c );
		}

		catalogAds.push_back( catalogAd );
	}

	classad::ExprList * ccList = new classad::ExprList( catalogAds );
	if(! ccList) {
		EXCEPT( "Shadow unable to allocate memory in UniShadow::start_staging_only_conversation().\n" );
	} else {
		commonCatalogsAd->Insert( "CommonCatalogsList", ccList );
	}


	ClassAd * colorAd = new ClassAd();
	colorAd->Insert( "CommonCatalogsAd", commonCatalogsAd );
	guidance.Insert( ATTR_COLOR_AD, dynamic_cast<ExprTree*>(colorAd) );

	request = co_yield guidance;

	success = false;
	LookupBoolInContext( request, ATTR_RESULT, success );
	if(! success) {
		// It's possible that a different starter would succeed,
		// so abort this attempt and leave the job in the queue idle.
		co_return VACATE_REQUEUE_ABORT(
			"Starter could not release execution resources after staging common files.",
			CONDOR_HOLD_CODE::JobNotStarted, JOB_NOT_STARTED_SUB_CODE::SlotColoringFailed
		);
	}


	std::string claimID;
	if(! LookupStringInContext( request, ATTR_SPLIT_CLAIM_ID, claimID )) {
		// It's possible that a different starter would reply properly,
		// so abort this attempt and leave the job in the queue idle.
		co_return VACATE_REQUEUE_ABORT(
			"Starter's reply invalid: did not contain split claim ID, aborting.",
			CONDOR_HOLD_CODE::JobNotStarted, JOB_NOT_STARTED_SUB_CODE::SlotColoringBadReply
		);
	}

	ClassAd * slotAd = NULL;
	if(! LookupAdInContext( request, ATTR_SLOT_AD, slotAd )) {
		// It's possible that a different starter would reply properly,
		// so abort this attempt and leave the job in the queue idle.
		co_return VACATE_REQUEUE_ABORT(
			"Starter's reply invalid: did not contain slot ad.",
			CONDOR_HOLD_CODE::JobNotStarted, JOB_NOT_STARTED_SUB_CODE::SlotColoringBadReply
		);
	}

	//
	// I don't know if this, strictly speaking, is supposed to be the
	// ATTR_USER or the ATTR_OWNER from the job ad, but it's consistently
	// referred to as the "submitter" in cmdDirectAttach(), and the job ad
	// doesn't include ATTR_SUBMITTER; one wonders what the startd was
	// originally intended to set this to.  A magic string from config?
	//
	// It turns out to be nontrivial to determine what the submitter
	// string ought to be, so just don't set this at all; if it isn't
	// set, and we send along a claim ID that the schedd recognizes,
	// it will pass that claim ID's match's `user` field as the `owner`
	// parameter to the MainScheddNegotiator object, which in turns
	// passes it down to the new match records, preventing them from
	// being empty and causing empty submitter strings from being sent
	// to the negotiator.
	//
	// This doesn't fix the problem for other callers of cmdDirectAttach(),
	// but I'd almost rather the engineering effort going into excising
	// 'owner', 'user', and 'submitter' from the codebase and replacing
	// them all with correct usages of a trio of arbitrary strings, just
	// so we have a chance of keeping thing straight.
	//
	std::string user;

/*
	long long sizeOnDiskInMB = -1;

	if(! LookupIntInContext( request, ATTR_COMMON_INPUT_FILES_SIZE_MB, sizeOnDiskInMB )) {
		// It's possible that a different starter would reply properly,
		// so abort this attempt and leave the job in the queue idle.
		co_return VACATE_REQUEUE_ABORT(
			"Starter's reply invalid: did not contain common input files size-on-disk.",
			CONDOR_HOLD_CODE::JobNotStarted, JOB_NOT_STARTED_SUB_CODE::SlotColoringBadReply
		);
	}
*/

	// We shouldn't need the address, since we're a shadow.
	DCSchedd schedd( getScheddAddr() /*, pool */ );
	if(! schedd.locate()) {
		EXCEPT( "start_staging_only_conversation(): failed to locate schedd.\n");
	}


	// This API makes me sad.
	char * originalClaimID = NULL;
	remRes->getClaimId( originalClaimID );
	if( originalClaimID == NULL ) {
		EXCEPT( "start_staging_only_conversation(): failed to acquire original claim ID.\n");
	}

	int timeout = 20;
	std::vector< std::pair< std::string, const ClassAd * > > resources;
	resources.emplace_back( claimID, slotAd );
	int rval = schedd.offerResources(
		resources, user, timeout,
		originalClaimID
		/*, sizeOnDiskInMB */
	);
	free( originalClaimID );

	if(! rval) {
		// Assuming that it's most likely that offerResources() failed
		// because the schedd was busy, we should try again later when
		// it isn't.
		co_return VACATE_REQUEUE_ABORT(
			"Failed to offer schedd resources, aborting to try again when it's less busy.\n",
			CONDOR_HOLD_CODE::JobNotStarted, JOB_NOT_STARTED_SUB_CODE::OfferResourcesFailed
		);
	}


	//
	// Now that we've staged the common files and let the schedd know,
	// it's keep-alive time.
	//
	while( true ) {
		guidance.Clear();
		guidance.InsertAttr(ATTR_COMMAND, COMMAND_RETRY_REQUEST);
		guidance.InsertAttr(ATTR_RETRY_DELAY, 300);
		dprintf( D_ALWAYS, "cxfer: Asking starter to sleep for five minutes...\n" );
		request = co_yield guidance;
	}
}


// The parameters are copies to simplify thinking about this coroutine.
condor::cr::Piperator<ClassAd, ClassAd>
UniShadow::start_mapping_only_conversation(
	ClassAd request,
	ListOfCatalogs common_file_catalogs,
	std::map<std::string, std::string> internalToSimpleNameMap
) {
	ClassAd guidance;


	// Map each catalog.  The schedd has guaranteed they were available.
	for( const auto & [cifName, commonInputFiles] : common_file_catalogs ) {
		dprintf( D_ZKM, "%s = %s\n", cifName.c_str(), commonInputFiles.c_str() );

		//
		// Determine the staging directory from the slot ad.  We don't want
		// to hard-code this determination in the starter because it may
		// change.  We could send a list of attribute-value pairs to the
		// starter to match against the slot ad, instead, but there's no
		// reason to do that in the starter instead of the shadow, and by
		// explicitly sending the cifName, we avoid adding more versioned
		// code to the starter (it won't try to figure it out on its own).
		//
		std::string stagingDir;

		// If I were feeling more ambitious, I would refactor the code from
		// start_staging_only_conversation() and call it here to fill in an
		// ad to do matching against.
		std::vector<std::pair<std::string, std::string>> requirements;
		requirements.emplace_back( ATTR_CATALOG, internalToSimpleNameMap[cifName] );

		{
			auto r = determineCIFScopeAndType( * jobAd );
			if(! r) { EXCEPT("Failed to determine CIF scope and type before mapping it."); }
			auto [scope, type] = * r;
			requirements.emplace_back( ATTR_CATALOG_SCOPE, scope );
			requirements.emplace_back( ATTR_CATALOG_SCOPE_TYPE, type );
		}

		std::string access_point;
		jobAd->LookupString( ATTR_GLOBAL_JOB_ID, access_point );
		access_point = split(access_point, "#")[0];
		requirements.emplace_back( ATTR_ACCESS_POINT, access_point );

		ClassAd * slotAd = remRes->getSlotAd();
		classad::ExprTree * e = slotAd ? slotAd->Lookup( "catalogs" ) : nullptr;
		classad::ExprList * l = dynamic_cast<classad::ExprList *>(e);

		if( slotAd == nullptr ) {
			dprintf( D_FULLDEBUG, "Starter did not send a slot ad, giving it a chance to find the catalog's path on its own.\n" );
		} else if( e == nullptr ) {
			dprintf( D_ALWAYS, "Starter sent a slot ad, but it didn't have a `catalogs` attribute; giving it a chance to find the catalog's path on its own.\n" );
		} else if( l == nullptr ) {
			dprintf( D_ALWAYS, "Started sent a slot ad, but its `catalogs` attribute wasn't a list; giving it a chance to find the catalog's path on its own.\n" );
		} else {
			for( auto i = l->begin(); i != l->end(); ++i ) {
				auto * ar = dynamic_cast<classad::AttributeReference *>(* i);
				if( ar == nullptr ) {
					dprintf( D_ALWAYS, "Entry in `catalogs` not an attribute reference, ignoring.\n" );
					continue;
				}

				classad::EvalState state;
				state.curAd = slotAd;
				classad::ExprTree * f = nullptr;
				classad::AttributeReference::Deref( * ar, state, f );
				if( f == nullptr ) {
					dprintf( D_ALWAYS, "Attribute reference in `catalogs` could not be dereferenced, ignoring.\n" );
					continue;
				}
				ClassAd * catalogAd = dynamic_cast<ClassAd *>(f);
				if( catalogAd == nullptr ) {
					dprintf( D_ALWAYS, "Attribute referenced in `catalogs` was not a ClassAd, ignoring.\n" );
					continue;
				}

				bool matches = true;
				for( auto [attribute, value] : requirements ) {
					classad::Value stringValue;
					std::string catalogValueProbe;
					formatstr( catalogValueProbe, "string(%s)", attribute.c_str() );
					if(! catalogAd->EvaluateExpr( catalogValueProbe, stringValue )) {
						dprintf( D_ALWAYS, "Could not evaluate %s.\n", catalogValueProbe.c_str() );
						matches = false;
						break;
					}

					std::string catalogValue;
					if(! stringValue.IsStringValue( catalogValue )) {
						dprintf( D_ALWAYS, "%s did not evaluate into a string.\n", catalogValueProbe.c_str() );
						matches = false;
						break;
					}

					if( value != catalogValue ) { matches = false; break; }
				}

				if( matches ) {
					catalogAd->LookupString( ATTR_CATALOG_PATH, stagingDir );
					dprintf( D_ALWAYS, "Found %s = %s\n", ATTR_CATALOG_PATH, stagingDir.c_str() );
					break;
				}
			}
		}

		guidance = do_wiring_up(stagingDir, cifName);
		request = co_yield guidance;

		bool success = false;
		LookupBoolInContext( request, ATTR_RESULT, success );
		if(! success) {
			co_return handle_wiring_failure();
		}
		Shadow->getJobAd()->InsertAttr( "CommonFilesMappedTime", time(NULL) );
	}


	guidance.Clear();
	guidance.InsertAttr(ATTR_COMMAND, COMMAND_CARRY_ON);
	co_return guidance;
}


// The parameters are copies to simplify thinking about this coroutine.
condor::cr::Piperator<ClassAd, ClassAd>
UniShadow::start_common_input_conversation(
	ClassAd request,
	ListOfCatalogs common_file_catalogs,
	bool print_waiting /* = true */
) {
	long long size = -1;
	ClassAd guidance;


	//
	// We'll handle the (cifName, object location list) tuples in order.
	// This means we can't `co_return` out of this function until we've
	// gone through the whole list.  That means we can't switch to the
	// uncommon input transfer until we've tried all the others.
	//
	// Check the git history for an extensive but now out-of-date
	// explanation of what we're doing here.
	//


	bool success;
	ListOfCatalogs unreadyCatalogs;
	for( const auto & [cifName, commonInputFiles] : common_file_catalogs ) {
		dprintf( D_ZKM, "%s = %s\n", cifName.c_str(), commonInputFiles.c_str() );

		SingleProviderSyndicate * cfLock = new SingleProviderSyndicate(cifName);
		auto [iter, inserted] = this->cfLocks.insert({cifName, cfLock});
		if(! inserted) {
			if( iter->first == cifName ) {
				dprintf( D_ALWAYS, "Ignoring duplicate catalog name '%s'\n", cifName.c_str() );
				continue;
			} else {
				EXCEPT("Failed to insert single provider syndicate into syndicate list, and it was not a duplicate catalog.\n" );
			}
		}

		std::string stagingDir;

		std::string message;
		auto status = cfLock->acquire( message );
		dprintf( D_ZKM, "start_common_input_conversation(): cfLock.acquire() = %d\n", (int)status );
		switch( status ) {
			//
			// Check each catalog:
			//   - if we're the provider, provide it immediately
			//   - if the catalog is ready, map it immediately
			//   - if the catalog is unready, record that and move on
			//
			// Once we exit the loop, we know that all catalogs are
			// either ready-and-mapped or unready.  If no catalogs are
			// unready, we're done: co_return CARRY_ON.  Otherwise,
			// we'd like to do input file transfer while we're waiting.
			// In that case, set this->resume_job_setup and co_yield CARRY_ON.
			//
			// When this coroutine resumes, it will loop over the unready
			// catalogs (as above); if any catalog remains unready after
			// the loop, it will co_yield RETRY_REQUEST, as currently.  It
			// will never set this->resume_job_setup because that's been
			// done already.
			//
			// When all the catalogs have been mapped, we co_return CARRY_ON.
			// (Once we've sent COMMAND_JOB_SETUP, which we do after setting
			// this->resume_job_setup and the starter asks for guidance after
			// doing input file transfer, the starter will no longer trigger
			// input file transfer; carrying on will just start the job.)
			//

			case SingleProviderSyndicate::PROVIDER:
				dprintf( D_TEST, "Producer elected.\n" );
				set_provider_keep_alive( cifName );

				// This would be cleaner as a nested coroutine, but not
				// so much so that I want to figure that out right now.
				guidance = before_common_file_transfer( cifName, commonInputFiles );
				request = co_yield guidance;
				success = after_common_file_transfer( request, cifName, stagingDir, size );
				if(! success) {
					guidance.Clear();
					guidance.InsertAttr(ATTR_COMMAND, COMMAND_ABORT);
					co_return guidance;
				}

				// We could reorder the cases to fall-through here...
				guidance = do_wiring_up( stagingDir, cifName );
				request = co_yield guidance;
				success = false;
				LookupBoolInContext( request, ATTR_RESULT, success );
				if(! success) {
					co_return handle_wiring_failure();
				}
				// Do NOT add CommonFilesMappedTime to the list of attributes
				// that gets passed through to the shadow.  It's only useful
				// for a specific job epoch.
				Shadow->getJobAd()->InsertAttr( "CommonFilesMappedTime", time(NULL) );

				break;

			case SingleProviderSyndicate::UNREADY:
				unreadyCatalogs.push_back({cifName, commonInputFiles});
				break;

			case SingleProviderSyndicate::READY:
				guidance = do_wiring_up(message, cifName);
				request = co_yield guidance;

				success = false;
				LookupBoolInContext( request, ATTR_RESULT, success );
				if(! success) {
					co_return handle_wiring_failure();
				}
				Shadow->getJobAd()->InsertAttr( "CommonFilesMappedTime", time(NULL) );
				break;

			case SingleProviderSyndicate::INVALID:
				EXCEPT("Something went terribly wrong in cfLock.acquire()." );
				break;

			case SingleProviderSyndicate::MIN:
				EXCEPT("Invalid return value (MIN) from cfLock.acquire()." );
				break;

			case SingleProviderSyndicate::MAX:
				EXCEPT("Invalid return value (MAX) from cfLock.acquire()." );
				break;
		} // end status switch
	} // end common_file_catalogs loop


	guidance.Clear();
	guidance.InsertAttr(ATTR_COMMAND, COMMAND_CARRY_ON);

	// If all the catalogs have been mapped, we have nothing more to do.
	if( unreadyCatalogs.empty() ) {
		dprintf( D_ZKM, "All catalogs mapped after the first pass.\n" );
		co_return guidance;
	}

	// Transfer input files while we're waiting for common file transfer.
	this->resume_job_setup = true;
	request = co_yield guidance;


	// I don't think we need to worry about this after the rewrite?
	bool job_environment_ready = false;
	LookupBoolInContext(
		request, ATTR_JOB_ENVIRONMENT_READY, job_environment_ready
	);
	dprintf( D_ZKM, "job_environment_ready = %s\n", job_environment_ready ? "true" : "false" );


	// Wait for (and deal with, if necessary) all the other common files.
	std::set< std::string > readyCatalogs;
	while(! unreadyCatalogs.empty()) {
		dprintf( D_ZKM, "Checking unready catalogs...\n" );

		// Check each unready catalog.  If it's READY or PROVIDER, do the
		// right thing and record the catalog as ready.
		for( const auto & [cifName, commonInputFiles] : unreadyCatalogs ) {
			dprintf( D_ZKM, "Unready catalog named '%s' lists '%s'.\n", cifName.c_str(), commonInputFiles.c_str() );

			SingleProviderSyndicate * cfLock = this->cfLocks[cifName];
			ASSERT(cfLock != NULL);
			std::string stagingDir;

			std::string message;
			auto status = cfLock->acquire( message );
			dprintf( D_ZKM, "start_common_input_conversation(): cfLock.acquire() = %d\n", (int)status );
			switch( status ) {
				case SingleProviderSyndicate::PROVIDER:
					dprintf( D_TEST, "Producer elected.\n" );
					set_provider_keep_alive( cifName );

					// This would be cleaner as a nested coroutine, but not
					// so much so that I want to figure that out right now.
					guidance = before_common_file_transfer( cifName, commonInputFiles );
					request = co_yield guidance;
					success = after_common_file_transfer( request, cifName, stagingDir, size );
					if(! success) {
						guidance.Clear();
						guidance.InsertAttr(ATTR_COMMAND, COMMAND_ABORT);
						co_return guidance;
					}

					// We could reorder the cases to fall-through here...
					guidance = do_wiring_up( stagingDir, cifName );
					request = co_yield guidance;
					success = false;
					LookupBoolInContext( request, ATTR_RESULT, success );
					if(! success) {
						co_return handle_wiring_failure();
					}
					Shadow->getJobAd()->InsertAttr( "CommonFilesMappedTime", time(NULL) );


					readyCatalogs.insert( cifName );
					break;

				case SingleProviderSyndicate::UNREADY:
					break;

				case SingleProviderSyndicate::READY:
					guidance = do_wiring_up(message, cifName);
					request = co_yield guidance;

					success = false;
					LookupBoolInContext( request, ATTR_RESULT, success );
					if(! success) {
						co_return handle_wiring_failure();
					}
					Shadow->getJobAd()->InsertAttr( "CommonFilesMappedTime", time(NULL) );

					readyCatalogs.insert( cifName );
					break;

				case SingleProviderSyndicate::INVALID:
					EXCEPT("Something went terribly wrong in cfLock.acquire()." );
					break;

				case SingleProviderSyndicate::MIN:
					EXCEPT("Invalid return value (MIN) from cfLock.acquire()." );
					break;

				case SingleProviderSyndicate::MAX:
					EXCEPT("Invalid return value (MAX) from cfLock.acquire()." );
					break;
			}
		}

		// The ready catalogs are no longer unready.
		dprintf( D_ZKM, "Found ready catalogs: %s.\n", join(readyCatalogs, ", ").c_str() );
		std::erase_if( unreadyCatalogs,
			[readyCatalogs](const ListOfCatalogs::value_type & k) {
				auto [cifName, commonInputFiles] = k;
				return readyCatalogs.contains(cifName);
			}
		);
		readyCatalogs.clear();
		dprintf( D_ZKM, "Unready catalogs are now: %s.\n", join(unreadyCatalogs, ", ").c_str() );

		// If there are any unready catalogs, check again in five
		// seconds after going back into the event loop.
		if(! unreadyCatalogs.empty()) {
			if( print_waiting ) {
				print_waiting = false;
				dprintf( D_ALWAYS, "Waiting for common files to be transferred (%s)\n", join(unreadyCatalogs, ", ").c_str() );

				CommonFilesEvent cfe;
				cfe.setType( CommonFilesEventType::WaitStarted );
				uLog.writeEvent( & cfe, jobAd );
			}
			guidance.InsertAttr(ATTR_COMMAND, COMMAND_RETRY_REQUEST);
			guidance.InsertAttr(ATTR_RETRY_DELAY, 5);
			// It's a little clumsy to send the starter's own context
			// ad back to it, but it's also easier than preserving it
			// across invocations over there.
			CopyAttribute(ATTR_CONTEXT_AD, guidance, request);
			request = co_yield guidance;
		}
	}

	guidance.Clear();
	guidance.InsertAttr(ATTR_COMMAND, COMMAND_CARRY_ON);
	co_return guidance;
}



extern bool use_guidance_in_job_ad;

GuidanceResult
send_guidance_from_job_ad( const ClassAd & /* request */, ClassAd & guidance ) {
	ExprTree * the_test_case = Shadow->getJobAd()->LookupExpr( "_condor_guidance_test_case" );
	if( the_test_case == NULL ) {
		guidance.InsertAttr(ATTR_COMMAND, COMMAND_CARRY_ON );
		return GuidanceResult::Command;
	}

	classad::ExprList * test_case_list = dynamic_cast<classad::ExprList *>(the_test_case);
	if( test_case_list == NULL ) {
		guidance.InsertAttr(ATTR_COMMAND, COMMAND_CARRY_ON );
		return GuidanceResult::Command;
	}

	std::vector<ExprTree *> the_list;
	test_case_list->GetComponents(the_list);

	// Track which ExprList we're iterating so the_index resets when the
	// shadow is recycled and picks up a new job (with a different list).
	static classad::ExprList * previous_list = nullptr;
	static int the_index = 0;
	if( test_case_list != previous_list ) {
		previous_list = test_case_list;
		the_index = 0;
	}
	ExprTree * the_guidance = the_list[the_index++];
	ClassAd * the_classad = dynamic_cast<ClassAd *>(the_guidance);
	if( the_classad == NULL ) {
		guidance.InsertAttr(ATTR_COMMAND, COMMAND_CARRY_ON );
		return GuidanceResult::Command;
	}
	guidance = * the_classad;
	return GuidanceResult::Command;
}


//
// This syscall MUST ignore information it doesn't know how to deal with.
//


GuidanceResult
BaseShadow::pseudo_request_guidance( const ClassAd & /* request */, ClassAd & guidance ) {
	dprintf( D_ALWAYS, "Internal error: BaseShadow::pseudo_request_guidance() called.\n" );
	guidance.InsertAttr( ATTR_COMMAND, COMMAND_CARRY_ON );
	return GuidanceResult::Command;
}


GuidanceResult
UniShadow::pseudo_request_guidance( const ClassAd & request, ClassAd & guidance ) {
	dprintf( D_ZKM, "Received request for guidance.\n" );

	if( param_boolean( "GUIDANCE_KEEP_CALM_AND", false ) ) {
		dprintf( D_ALWAYS, "Keep calm and (always send the command) %s\n", COMMAND_CARRY_ON );
		guidance.InsertAttr( ATTR_COMMAND, COMMAND_CARRY_ON );
		return GuidanceResult::Command;
	}

	std::string requestType;
	if(! request.LookupString( ATTR_REQUEST_TYPE, requestType )) {
		return GuidanceResult::MalformedRequest;
	}

	if( requestType == RTYPE_JOB_ENVIRONMENT ) {
		dprintf( D_ZKM, "Received request for guidance about the job environment.\n" );

		// There's no reason for this to be exclusive to the job environment
		// guidance, but to unbreak the test, let's pretend it is.  (The
		// in-job guidance would have to specify the request type.)
		if( use_guidance_in_job_ad ) {
			dprintf( D_TEST, "Using guidance in job ad.\n" );
			return send_guidance_from_job_ad( request, guidance );
		}

		if( thisRemoteResource->download_transfer_info.xfer_status == XFER_STATUS_UNKNOWN ) {
			// This isn't copied into thisRemoteResource->download_transfer_info
			// until the FTO reaper fires, which might be a a while.
			const FileTransfer::FileTransferInfo current = thisRemoteResource->filetrans.GetInfo();
			if ( current.xfer_status == XFER_STATUS_ACTIVE ) {
				// We haven't gotten an update from our side's FTO yet.  What
				// I would like to do is go back into the event loop and re-
				// check after a few seconds, but the syscall socket code is
				// _very_ synchronous.  Instead, since we'll want to have
				// this command for other purposes later, ask the starter to
				// call us back in a few seconds.
				//
				// Because we're now checking the live copy of the FTO status
				// variables, this should never happen, but since we have a
				// coded work-around and it's still at least possible, let's
				// leave it in.
				guidance.InsertAttr(ATTR_COMMAND, COMMAND_RETRY_REQUEST);
				guidance.InsertAttr(ATTR_RETRY_DELAY, 5);
				// guidance.InsertAttr(ATTR_CONTEXT_AD, context);
			} else if( current.xfer_status == XFER_STATUS_DONE
			 && current.success == true
			) {
				if(! this->resume_job_setup) {
					guidance.InsertAttr(ATTR_COMMAND, COMMAND_START_JOB);
				} else {
					this->resume_job_setup = false;
					guidance.InsertAttr(ATTR_COMMAND, COMMAND_JOB_SETUP);
				}
			} else if (
				current.xfer_status == XFER_STATUS_DONE
			 && current.success == false
			) {
				if( current.try_again == true ) {
					// I'm not sure this case ever actually happens in practice,
					// but in case it does, this seems like the right thing to do.
					guidance.InsertAttr(ATTR_COMMAND, COMMAND_RETRY_TRANSFER);
				} else {
					//
					// It's massive overkill for a simple two-step protocol, but
					// let's make this a coroutine so that we can easily make it
					// more complicated.
					//
					// Since we're only talking to one starter at a time, we can
					// simply record if we've already started this conversation.
					//
					static bool in_conversation = false;
					static condor::cr::Piperator<ClassAd, ClassAd> the_coroutine;

					if(! in_conversation) {
						dprintf( D_ZKM, "Starting common input files conversation during environment setup.\n" );
						in_conversation = true;
						the_coroutine = std::move(
							start_input_transfer_failure_conversation(request)
						);
						guidance = the_coroutine();
					} else {
						dprintf( D_ZKM, "Continuing common input files conversation during environment setup.\n" );
						the_coroutine.set_co_yield_value( request );
						guidance = the_coroutine();
					}

					if( the_coroutine.handle.done() ) {
						dprintf( D_ZKM, "Finishing common input files conversation during environment setup.\n" );
						in_conversation = false;
					}
				}
			} else {
				guidance.InsertAttr( ATTR_COMMAND, COMMAND_CARRY_ON );
			}
		} else {
			// The starter asked for guidance about the job environment, but
			// we think we're doing output transfer.
			guidance.InsertAttr( ATTR_COMMAND, COMMAND_CARRY_ON );
		}

		std::string command;
		guidance.LookupString(ATTR_COMMAND, command);
		dprintf( D_ZKM, "Sending (job environment) guidance with command %s\n", command.c_str());
		return GuidanceResult::Command;
	} else if( requestType == RTYPE_JOB_SETUP ) {
		dprintf( D_ZKM, "Received request for guidance about job setup.\n" );

		//
		// The schedd has already decided for us if we're going to stage
		// common files, xor map them, xor fall back, but we'll also
		// continue to respect the config flag for safety.
		//
		bool disallowed = param_boolean("FORBID_COMMON_FILE_TRANSFER", false);
		if( disallowed || cxfer_type == CXFER_STATE::INVALID ) {
			guidance.InsertAttr( ATTR_COMMAND, COMMAND_CARRY_ON );

			std::string command;
			guidance.LookupString(ATTR_COMMAND, command);
			dprintf( D_TEST, "Sending (job setup) guidance with command %s\n", command.c_str());
			return GuidanceResult::Command;
		}


		int required_version = 2;
		std::map<std::string, std::string> internalToSimpleNameMap;
		auto common_file_catalogs = computeCommonInputFileCatalogs( jobAd, & internalToSimpleNameMap );
		if(! common_file_catalogs) {
			dprintf( D_ERROR, "Failed to compute common input file catalogs, can't run job!\n" );

			// We don't have a mechanism to inform the submitter of internal
			// errors like this, so for now we're stuck putting the job on hold.
			holdJob( "Internal error: failed to compute common input file catalogs.",
				CONDOR_HOLD_CODE::JobNotStarted, JOB_NOT_STARTED_SUB_CODE::CatalogNameError
			);

			guidance.InsertAttr( ATTR_COMMAND, COMMAND_ABORT );
			return GuidanceResult::Command;
		}

		if(! computeCommonInputFiles( jobAd, *common_file_catalogs, required_version, & internalToSimpleNameMap )) {
			dprintf( D_ERROR, "Failed to compute common input files, can't run job!\n" );
			// We don't have a mechanism to inform the submitter of internal
			// errors like this, so for now we're stuck putting the job on hold.
			holdJob( "Internal error: failed to compute common input files",
				CONDOR_HOLD_CODE::JobNotStarted, JOB_NOT_STARTED_SUB_CODE::CatalogNameError
			);

			guidance.InsertAttr( ATTR_COMMAND, COMMAND_ABORT );
			return GuidanceResult::Command;
		}

		//
		// The schedd may have decided that we don't need to transfer every
		// catalog that the job requires.  Check for `TransferTheseCatalogs`.
		//
		// I tried refactoring this to use std::[ranges::]set_intersection().
		// The required sorting made the code rather ugly, but it turns out
		// that set_intersection() has an (effectively) undocumented
		// requirement that the types of the two ranges be the same.  (The
		// non-ranges implementation calls the comparator twice, with the
		// left- and right- hand sides swapped, but only allows you to supply
		// one; the ranges implementation requires that the elements of both
		// ranges be assignable to the output range, even though there's no
		// need to copy from both ranges.)
		//

		auto transfer_these_catalogs = LookupClassAdListOfStrings(
			* jobAd, ATTR_TRANSFER_THESE_CATALOGS
		);
		if( transfer_these_catalogs ) {
			ListOfCatalogs filtered;

			for( const auto & name : * transfer_these_catalogs ) {
				auto in = std::find_if(
					(* common_file_catalogs).begin(),
					(* common_file_catalogs).end(),
					[name](const auto & e) { return e.first == name; }
				);
				if( in != common_file_catalogs->end() ) {
					filtered.push_back( * in );
				}
			}

			* common_file_catalogs = filtered;
		}


		//
		// Since we're only talking to one starter at a time, we can
		// simply record if we've already started this conversation.
		//
		static bool in_conversation = false;
		static condor::cr::Piperator<ClassAd, ClassAd> the_coroutine;

		if(! in_conversation) {
			dprintf( D_ZKM, "Starting common input files conversation during job setup.\n" );
			in_conversation = true;

			switch( cxfer_type ) {
				case CXFER_STATE::INVALID:
					ASSERT(cxfer_type != CXFER_STATE::INVALID);
					break;

				case CXFER_STATE::STAGING:
					the_coroutine = std::move(
						this->start_staging_only_conversation(request, *common_file_catalogs, internalToSimpleNameMap)
					);
					break;

				case CXFER_STATE::MAPPING:
					the_coroutine = std::move(
						this->start_mapping_only_conversation(request, *common_file_catalogs, internalToSimpleNameMap)
					);
					break;

				case CXFER_STATE::STAGED:
					ASSERT(cxfer_type != CXFER_STATE::STAGED);
					break;

				case CXFER_STATE::RETIRING:
					ASSERT(cxfer_type != CXFER_STATE::RETIRING);
				    break;
			}

			guidance = the_coroutine();
		} else {
			dprintf( D_ZKM, "Continuing common input files conversation during job setup.\n" );
			the_coroutine.set_co_yield_value( request );
			guidance = the_coroutine();
		}

		if( the_coroutine.handle.done() ) {
			dprintf( D_ZKM, "Finishing common input files conversation during job setup.\n" );
			in_conversation = false;
		}

		std::string command;
		guidance.LookupString(ATTR_COMMAND, command);
		dprintf( D_TEST, "Sending (job setup) guidance with command %s\n", command.c_str());
		return GuidanceResult::Command;
	}

	return GuidanceResult::UnknownRequest;
}


std::optional<ListOfCatalogs>
UniShadow::computeCommonInputFileCatalogs(
	ClassAd * jobAd,
	std::map<std::string, std::string> * internalToSimpleNameMap
) {
	char * startdAddress = NULL;
	this->remRes->getStartdAddress(startdAddress);
	auto rval = ::computeCommonInputFileCatalogs( jobAd, startdAddress, internalToSimpleNameMap );
	free( startdAddress );
	return rval;
}


bool
UniShadow::computeCommonInputFiles(
	ClassAd * jobAd,
	ListOfCatalogs & commonFileCatalogs,
	int & required_version,
	std::map<std::string, std::string> * internalToSimpleNameMap
) {
	char * startdAddress = NULL;
	this->remRes->getStartdAddress(startdAddress);
	auto rval = ::computeCommonInputFiles( jobAd, startdAddress, commonFileCatalogs, required_version, internalToSimpleNameMap );
	free( startdAddress );
	return rval;
}
