Is it not possible to restore from GPU 0 to GPU 0?

[wangzhen2271]

Is it possible to restore to the same GPU device?

[rst0git]

@wangzhen2271 I believe this is the default functionality. The cuda-checkpoint tool has --device-map option that allows to restore on different devices, but by default, it should restore GPU 0 to GPU 0.

--device-map|-d <deviceUuids>
	Optional device map used during the restore action to manually remap old devices onto new ones.
	<deviceUuids> is a comma delimited list in the format oldUuid1=newUuid1,oldUuid2=newUuid2,...
	Must contain all checkpointed devices if specified.

Would it be possible to provide more information? Do you see any errors?

[wangzhen2271]

int cuda_gpu_migrate(pid_t target_pid, int src_gpu, int dst_gpu)
{
int dev_count;
int ret = 0;
cuDeviceGetCount(&dev_count);
if (src_gpu < 0 || src_gpu >= dev_count || dst_gpu < 0 || dst_gpu >= dev_count)
{
fprintf(stderr, "Invalid GPU index! src=%d, dst=%d, total GPU=%d\n", src_gpu, dst_gpu, dev_count);
return -1;
}

CUcheckpointGpuPair *pairs = (CUcheckpointGpuPair *)calloc(1, sizeof(CUcheckpointGpuPair));
if (pairs == NULL)
{
    perror("calloc pairs failed");
    return -1;
}

 ret = cuDeviceGetUuid(&pairs[0].oldUuid, src_gpu);
    if (ret != 0)
    {
        printf("get  GPU %d uuid failed\n", src_gpu);
        return ret;
    }
    ret = cuDeviceGetUuid(&pairs[0].newUuid,dst_gpu);
    if (ret != 0)
    {
        printf("get  GPU %d uuid failed\n", dst_gpu);
        return ret;
    }


CUcheckpointLockArgs lock_args = {0};
CUcheckpointCheckpointArgs checkpoint_args = {0};
CUcheckpointRestoreArgs restore_args = {0};
CUcheckpointUnlockArgs unlock_args = {0};


restore_args.gpuPairsCount = 1;
restore_args.gpuPairs = pairs;

fprintf(stdout, "Start migrate GPU %d -> %d for PID %d...\n", src_gpu, dst_gpu, target_pid);

ret = cuCheckpointProcessLock(target_pid, &lock_args);
if (ret != 0)
{
    printf("lock process %d failed\n", target_pid);
    return ret;
}
ret = cuCheckpointProcessCheckpoint(target_pid, &checkpoint_args);
if (ret != 0)
{
    printf("Checkpoint process %d failed\n", target_pid);
    return ret;
}
getchar();
ret = cuCheckpointProcessRestore(target_pid, &restore_args);
if (ret != 0)
{
    printf("Restore process %d failed ret %d\n", target_pid, ret);
    return ret;
}
ret = cuCheckpointProcessUnlock(target_pid, &unlock_args);

fprintf(stdout, "Migrate PID %d GPU %d -> %d success!\n", target_pid, src_gpu, dst_gpu);


free(pairs);
return 0;

}
The cuCheckpointProcessRestore function fails with the code I've written this way. I passed 0 to 0 as the arguments.

[wangzhen2271]

@rst0git
Do you know how to do it?