Attachment #445556 for bug #593624




#else
#define NV_KMEM_CACHE_CREATE_FULL(name, size, align, flags, ctor) \
    kmem_cache_create(name, size, align, flags, ctor, NULL)
#endif

#define NV_KMEM_CACHE_CREATE(name, type)    \
    NV_KMEM_CACHE_CREATE_FULL(name, sizeof(type), 0, 0, NULL)

#define NV_KMEM_CACHE_CREATE_USERCOPY(name, type)    \
    NV_KMEM_CACHE_CREATE_FULL(name, sizeof(type), 0, SLAB_USERCOPY, NULL)

#define NV_KMEM_CACHE_DESTROY(kmem_cache)   \
    kmem_cache_destroy(kmem_cache)

#define NV_KMEM_CACHE_ALLOC(kmem_cache)     \
    kmem_cache_alloc(kmem_cache, GFP_KERNEL)
#define NV_KMEM_CACHE_FREE(ptr, kmem_cache) \
    kmem_cache_free(kmem_cache, ptr)





typedef struct {
    /*
     * The nvidia-modeset kernel module should assign version_string
     * before passing the structure to the nvidia kernel module, so
     * that a version match can be confirmed: it is not supported to
     * mix nvidia and nvidia-modeset kernel modules from different
     * releases.
     */
    // const char *version_string;

    /*
     * Return system information.
     */
    struct {
        /* Availability of write combining support for video memory */
        NvBool allow_write_combining;
    } system_info;


    void NV_MODESET_INTERFACE_API_CALL (*op)
        (nvidia_modeset_stack_ptr sp, void *ops_cmd);

    int (*set_callbacks)(const nvidia_modeset_callbacks_t *cb);

} nvidia_modeset_rm_ops_t;

NV_STATUS nvidia_get_rm_ops(const nvidia_modeset_rm_ops_t **rm_ops, const char **version_string);

#endif /* _NV_MODESET_INTERFACE_H_ */





    /* file operations */
    int (*open)(struct inode *, struct file *filp);
    int (*close)(struct inode *, struct file *filp);
    int (*mmap)(struct file *filp, struct vm_area_struct *vma);
    int (*ioctl)(struct inode *, struct file * file, unsigned int cmd, unsigned long arg);
    unsigned int (*poll)(struct file * file, poll_table *wait);

} __do_const nvidia_module_t;

int nvidia_register_module(nvidia_module_t *);
int nvidia_unregister_module(nvidia_module_t *);

#endif





#if defined(VM_CHECKER)
    NV_SPIN_LOCK_INIT(&vm_lock);
#endif
#if defined(KM_CHECKER)
    NV_SPIN_LOCK_INIT(&km_lock);
#endif

    nvidia_stack_t_cache = NV_KMEM_CACHE_CREATE_USERCOPY(nvidia_stack_cache_name,
                                                         nvidia_stack_t);
    if (nvidia_stack_t_cache == NULL)
    {
        nv_printf(NV_DBG_ERRORS, "NVRM: stack cache allocation failed!\n");
        rc = -ENOMEM;
        goto failed5;
    }

    rc = nv_kmem_cache_alloc_stack(&sp);




#include "nv-linux.h"
#include "nv-frontend.h"
#include "nv-instance.h"

int nv_register_chrdev(void *param)
{
    nvidia_module_t *module = (nvidia_module_t *)param;

    module->instance = nv_module_instance;

    return (nvidia_register_module(module));
}

void nv_unregister_chrdev(void *param)
{
    nvidia_module_t *module = (nvidia_module_t *)param;

    nvidia_unregister_module(module);




    .err_handler = &nv_pci_error_handlers,
#endif
};

/* character device entry points*/
nvidia_module_t nv_fops = {
    .owner       = THIS_MODULE,
    .module_name = MODULE_NAME,
    .instance    = MODULE_INSTANCE_NUMBER,
    .open        = nvidia_open,
    .close       = nvidia_close,
    .ioctl       = nvidia_ioctl,
    .mmap        = nvidia_mmap,
    .poll        = nvidia_poll,
};

/*




            nvfp = NV_GET_FILE_PRIVATE(NV_VMA_FILE(vma));
            at->next = nvfp->free_list;
            nvfp->free_list = at;
        }
    }
}

#if defined(NV_VM_OPERATIONS_STRUCT_HAS_ACCESS)
static ssize_t
nvidia_vma_access(
    struct vm_area_struct *vma,
    unsigned long addr,
    void *buffer,
    size_t length,
    int write
)
{
    nv_alloc_t *at = NULL;
    nv_file_private_t *nvfp = NV_GET_FILE_PRIVATE(NV_VMA_FILE(vma));
    nv_state_t *nv = NV_STATE_PTR(nvfp->nvptr);
    NvU32 pageIndex, pageOffset;
    void *kernel_mapping;




        count++;
    }

    UNLOCK_NV_LINUX_DEVICES();

    return count;
}

NV_STATUS nvidia_get_rm_ops(const nvidia_modeset_rm_ops_t **rm_ops, const char **version_string)
{
    static nvidia_modeset_rm_ops_t local_rm_ops = {
        .version_string = NV_VERSION_STRING,
        .system_info    = {
            .allow_write_combining = NV_FALSE,
        },
        .alloc_stack    = nvidia_modeset_rm_ops_alloc_stack,
        .free_stack     = nvidia_modeset_rm_ops_free_stack,
        .enumerate_gpus = nvidia_modeset_enumerate_gpus,
        .open_gpu       = nvidia_dev_get,
        .close_gpu      = nvidia_dev_put,
        .op             = rm_kernel_rmapi_op, /* provided by nv-kernel.o */
        .set_callbacks  = nvidia_modeset_set_callbacks,
    };

    if (strcmp(*version_string, NV_VERSION_STRING) != 0)
    {
        *version_string = NV_VERSION_STRING;
        return NV_ERR_GENERIC;
    }

    *rm_ops = (const nvidia_modeset_rm_ops_t *) &local_rm_ops;

    if (NV_ALLOW_WRITE_COMBINING(NV_MEMORY_TYPE_FRAMEBUFFER)) {
        local_rm_ops.system_info.allow_write_combining = NV_TRUE;
    }

    return NV_OK;
}

EXPORT_SYMBOL(nvidia_get_rm_ops);




    DRM_IOCTL_DEF_DRV(NVIDIA_GEM_PRIME_FENCE_FORCE_SIGNAL,
                      nvidia_drm_gem_prime_fence_force_signal,
                      DRM_CONTROL_ALLOW|DRM_RENDER_ALLOW|DRM_UNLOCKED),
    DRM_IOCTL_DEF_DRV(NVIDIA_GEM_PRIME_FENCE_FINI,
                      nvidia_drm_gem_prime_fence_fini,
                      DRM_CONTROL_ALLOW|DRM_RENDER_ALLOW|DRM_UNLOCKED),
};

static drm_driver_no_const nv_drm_driver __read_only  = {

    .driver_features        = DRIVER_GEM | DRIVER_PRIME | DRIVER_RENDER,

    .gem_free_object        = nvidia_drm_gem_free,

    .ioctls                 = nv_drm_ioctls,
    .num_ioctls             = ARRAY_SIZE(nv_drm_ioctls),


{
#if defined(NV_DRM_ATOMIC_MODESET_AVAILABLE)

    if (!nvidia_drm_modeset_module_param)
    {
        return;
    }

    pax_open_kernel();
    nv_drm_driver.driver_features |= DRIVER_MODESET | DRIVER_ATOMIC;

    nv_drm_driver.master_set       = nvidia_drm_master_set;
    nv_drm_driver.master_drop      = nvidia_drm_master_drop;

    nv_drm_driver.dumb_create      = nvidia_drm_dumb_create;
    nv_drm_driver.dumb_map_offset  = nvidia_drm_dumb_map_offset;
    nv_drm_driver.dumb_destroy     = drm_gem_dumb_destroy;
    pax_close_kernel();

#endif /* NV_DRM_ATOMIC_MODESET_AVAILABLE */
}



/*
 * Helper function for allocate/register DRM device for given NVIDIA GPU ID.




    up(&nvkms_lock);
}


/*************************************************************************
 * Interface with resman.
 *************************************************************************/

static const nvidia_modeset_rm_ops_t *__rm_ops;

static nvidia_modeset_callbacks_t nvkms_rm_callbacks = {
    .suspend = nvkms_suspend,
    .resume = nvkms_resume
};

static int nvkms_alloc_rm(void)
{
    NV_STATUS nvstatus;
    int ret;
    const char * version_string = NV_VERSION_STRING;

    nvstatus = nvidia_get_rm_ops(&__rm_ops, &version_string);

    nvstatus = nvidia_get_rm_ops(&__rm_ops);

    if (nvstatus != NV_OK) {
        printk(KERN_ERR NVKMS_LOG_PREFIX "Version mismatch: "
               "nvidia.ko(%s) nvidia-modeset.ko(%s)\n",
	       version_string, NV_VERSION_STRING);
        return -EINVAL;
    }

    ret = __rm_ops->set_callbacks(&nvkms_rm_callbacks);
    if (ret < 0) {
        printk(KERN_ERR NVKMS_LOG_PREFIX "Failed to register callbacks\n");
        return ret;
    }

    return 0;
}

static void nvkms_free_rm(void)
{
    __rm_ops->set_callbacks(NULL);
}

void NVKMS_API_CALL nvkms_call_rm(void *ops)
{
    nvidia_modeset_stack_ptr stack = NULL;

    if (__rm_ops->alloc_stack(&stack) != 0) {
        return;
    }

    __rm_ops->op(stack, ops);

    __rm_ops->free_stack(stack);
}

/*************************************************************************
 * ref_ptr implementation.
 *************************************************************************/

struct nvkms_ref_ptr {
    struct kref refcnt;

    return data;
}

NvBool NVKMS_API_CALL nvkms_open_gpu(NvU32 gpuId)
{
    nvidia_modeset_stack_ptr stack = NULL;
    NvBool ret;

    if (__rm_ops->alloc_stack(&stack) != 0) {
        return NV_FALSE;
    }

    ret = __rm_ops->open_gpu(gpuId, stack) == 0;

    __rm_ops->free_stack(stack);

    return ret;
}

void NVKMS_API_CALL nvkms_close_gpu(NvU32 gpuId)
{
    nvidia_modeset_stack_ptr stack = NULL;

    if (__rm_ops->alloc_stack(&stack) != 0) {
        return;
    }

    __rm_ops->close_gpu(gpuId, stack);

    __rm_ops->free_stack(stack);
}

NvU32 NVKMS_API_CALL nvkms_enumerate_gpus(nv_gpu_info_t *gpu_info)
{
    return __rm_ops->enumerate_gpus(gpu_info);
}

NvBool NVKMS_API_CALL nvkms_allow_write_combining(void)
{
    return __rm_ops->system_info.allow_write_combining;
}

/*************************************************************************
 * Common to both user-space and kapi NVKMS interfaces
 *************************************************************************/

static void nvkms_kapi_event_work_queue_callback(NVKMS_WORK_FUNC_ARG_T *arg)
{




#include "uvm8_thread_context.h"
#include "uvm8_va_range.h"
#include "uvm8_kvmalloc.h"
#include "uvm8_mmu.h"
#include "uvm8_perf_heuristics.h"
#include "nv_uvm_interface.h"

uvm_global_t g_uvm_global;
static struct UvmOpsUvmEvents g_exported_uvm8_ops = {
    .startDevice = NULL,
    .stopDevice  = NULL,
    .isrTopHalf  = uvm8_isr_top_half,
};
static bool g_ops_registered = false;

static NV_STATUS uvm8_register_callbacks(void)
{
    NV_STATUS status = NV_OK;

    g_exported_uvm8_ops.startDevice = NULL;
    g_exported_uvm8_ops.stopDevice  = NULL;
    g_exported_uvm8_ops.isrTopHalf  = uvm8_isr_top_half;

    // Register the UVM callbacks with the main GPU driver:
    status = uvm_rm_locked_call(nvUvmInterfaceRegisterUvmCallbacks(&g_exported_uvm8_ops));
    if (status != NV_OK)
        return status;

    g_ops_registered = true;
    return NV_OK;
}




{
    NvU32 index = get_index(semaphore);
    NvU64 base_va = uvm_rm_mem_get_gpu_va(semaphore->page->memory, gpu);
    return base_va + UVM_SEMAPHORE_SIZE * index;
}

NvU32 uvm_gpu_semaphore_get_payload(uvm_gpu_semaphore_t *semaphore)
{
    return ACCESS_ONCE_RW(*semaphore->payload);
}

void uvm_gpu_semaphore_set_payload(uvm_gpu_semaphore_t *semaphore, NvU32 payload)
{
    // Provide a guarantee that all memory accesses prior to setting the payload
    // won't be moved past it.
    // Use a big hammer mb() as set_payload() is not used in any performance path
    // today.
    // This could likely be optimized to be either an smp_store_release() or use
    // an smp_mb__before_atomic() barrier. The former is a recent addition to
    // kernel though, and it's not clear whether combining the latter with a
    // regular 32bit store is well defined in all cases. Both also seem to risk
    // being optimized out on non-SMP configs (we need them for interacting with
    // the GPU correctly even on non-SMP).
    mb();
    ACCESS_ONCE_RW(*semaphore->payload) = payload;
}

// This function is intended to catch channels which have been left dangling in
// trackers after their owning GPUs have been destroyed.
static bool tracking_semaphore_check_gpu(uvm_gpu_tracking_semaphore_t *tracking_sem)
{
    uvm_gpu_t *gpu = tracking_sem->semaphore.page->pool->gpu;
    uvm_gpu_t *table_gpu;




        uvm_ce_hal_t ce_ops;

        // arch_ops: id is an architecture
        uvm_arch_hal_t arch_ops;

        // fault_buffer_ops: id is a hardware class
        uvm_fault_buffer_hal_t fault_buffer_ops;
    } u;
} __do_const uvm_hal_class_ops_t;

// When UVM next support is enabled support for future chips in the hal is
// enabled by providing additional hal table entries below.
#if UVM_IS_NEXT()
  extern uvm_hal_class_ops_t *ce_table_next;
  extern uvm_hal_class_ops_t *host_table_next;
  extern uvm_hal_class_ops_t *arch_table_next;
  extern uvm_hal_class_ops_t *fault_buffer_table_next;




    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
    DEALINGS IN THE SOFTWARE.

*******************************************************************************/

#ifndef __UVM8_MMU_H__
#define __UVM8_MMU_H__

#include "uvm8_forward_decl.h"
#include "uvm8_hal_types.h"
#include "uvm8_pmm_gpu.h"
#include "uvmtypes.h"
#include "uvm_common.h"
#include "uvm8_tracker.h"
#include "uvm8_test_ioctl.h"

// Used when the page size isn't known and should not matter




#include "uvm8_init.h"
#include "uvm8_forward_decl.h"

// TODO: Bug 1710855: Tweak this number through benchmarks
#define UVM_SPIN_LOOP_SCHEDULE_TIMEOUT_NS   (10*1000ULL)
#define UVM_SPIN_LOOP_PRINT_TIMEOUT_SEC     30ULL

static dev_t g_uvmBaseDev;
struct UvmOpsUvmEvents g_exportedUvmOps;

static char* uvm_driver_mode = "8";

// UVM-Lite behavior is activated through insmod argument: (default is "8")
//      insmod nvidia-uvm.ko uvm_driver_mode="lite"
//
// Similarly, the UVM-8 production driver can be activated with:
//      insmod nvidia-uvm.ko uvm_driver_mode="8"


    return nvStatus;
}

static NV_STATUS uvmSetupGpuProvider(void)
{
    NV_STATUS status = NV_OK;

    static struct UvmOpsUvmEvents g_exportedUvmOps = {
        .startDevice = uvm_gpu_event_start_device,
        .stopDevice  = uvm_gpu_event_stop_device,
    };

    // call RM to exchange the function pointers.
    status = nvUvmInterfaceRegisterUvmCallbacks(&g_exportedUvmOps);
    return status;
}

static int uvm_not8_init(dev_t uvmBaseDev)
{




// uvm_full_fault_buffer.h
//
// This file contains structures and function declarations to read or update
// the fault buffer/related registers and read/mask/unmask the fault intr.
//
#ifndef _UVM_FULL_FAULT_BUFFER_H_
#define _UVM_FULL_FAULT_BUFFER_H_

#include <linux/compiler.h>
#include "uvmtypes.h"

#define MAXWELL_FAULT_BUFFER_A (0xb069)
#define MEM_RD32(a) (*(const volatile NvU32 *)(a)) 
#define MEM_WR32(a, d) do { *(volatile NvU32 *)(a) = (d); } while (0)
typedef enum
{
    UvmReplayType_none,

    NvUvmSetReplayParamsReg_t           setReplayParamsReg;
    NvUvmGetFaultPacketSize_t           getFaultPacketSize;
    NvUvmWriteFaultBufferPacket_t       writeFaultBufferPacket;
    NvUvmIsFaultInterruptPending_t      isFaultIntrPending;
    NvUvmSetFaultIntrBit_t              setFaultIntrBit;
    NvUvmControlPrefetch_t              controlPrefetch;
    NvUvmTestFaultBufferOverflow_t      testFaultBufferOverflow;
    NvUvmClearFaultBufferOverflow_t     clearFaultBufferOverflow;
} __no_const UvmFaultBufferOps;

/******************************************************************************
    uvmfull_fault_buffer_init
        Initialze fault buffer management related function pointers for a 
        fault class

    Arguments:
        faultBufferClass: (INPUT)





// Added in 2.6.24
#ifndef ACCESS_ONCE
  #define ACCESS_ONCE(x) (*(volatile typeof(x) *)&(x))
#endif

// WRITE_ONCE/READ_ONCE have incompatible definitions across versions, which produces warnings.
// Therefore, we define our own macros
#define UVM_WRITE_ONCE(x, val) (ACCESS_ONCE_RW(x) = (val))
#define UVM_READ_ONCE(x) ACCESS_ONCE(x)

// Added in 3.11
#ifndef PAGE_ALIGNED
    #define PAGE_ALIGNED(addr) (((addr) & (PAGE_SIZE - 1)) == 0)
#endif

// Added in 2.6.37 via commit e1ca7788dec6773b1a2bce51b7141948f2b8bccf

Return to bug 593624

Lines 655-672 Link Here

(-)kernel_/nvidia/nv.c (-2 / +2 lines)
655		655
656	#if defined(VM_CHECKER)	656	#if defined(VM_CHECKER)
657	NV_SPIN_LOCK_INIT(&vm_lock);	657	NV_SPIN_LOCK_INIT(&vm_lock);
658	#endif	658	#endif
659	#if defined(KM_CHECKER)	659	#if defined(KM_CHECKER)
660	NV_SPIN_LOCK_INIT(&km_lock);	660	NV_SPIN_LOCK_INIT(&km_lock);
661	#endif	661	#endif
662		662
663	nvidia_stack_t_cache = NV_KMEM_CACHE_CREATE(nvidia_stack_cache_name,	663	nvidia_stack_t_cache = NV_KMEM_CACHE_CREATE_USERCOPY(nvidia_stack_cache_name,
664	nvidia_stack_t);	664	nvidia_stack_t);
665	if (nvidia_stack_t_cache == NULL)	665	if (nvidia_stack_t_cache == NULL)
666	{	666	{
667	nv_printf(NV_DBG_ERRORS, "NVRM: stack cache allocation failed!\n");	667	nv_printf(NV_DBG_ERRORS, "NVRM: stack cache allocation failed!\n");
668	rc = -ENOMEM;	668	rc = -ENOMEM;
669	goto failed5;	669	goto failed5;
670	}	670	}
671		671
672	rc = nv_kmem_cache_alloc_stack(&sp);	672	rc = nv_kmem_cache_alloc_stack(&sp);

Lines 108-129 Link Here

(-)kernel_/nvidia/nv-mmap.c (-2 / +2 lines)
108	nvfp = NV_GET_FILE_PRIVATE(NV_VMA_FILE(vma));	108	nvfp = NV_GET_FILE_PRIVATE(NV_VMA_FILE(vma));
109	at->next = nvfp->free_list;	109	at->next = nvfp->free_list;
110	nvfp->free_list = at;	110	nvfp->free_list = at;
111	}	111	}
112	}	112	}
113	}	113	}
114		114
115	#if defined(NV_VM_OPERATIONS_STRUCT_HAS_ACCESS)	115	#if defined(NV_VM_OPERATIONS_STRUCT_HAS_ACCESS)
116	static int	116	static ssize_t
117	nvidia_vma_access(	117	nvidia_vma_access(
118	struct vm_area_struct *vma,	118	struct vm_area_struct *vma,
119	unsigned long addr,	119	unsigned long addr,
120	void *buffer,	120	void *buffer,
121	int length,	121	size_t length,
122	int write	122	int write
123	)	123	)
124	{	124	{
125	nv_alloc_t *at = NULL;	125	nv_alloc_t *at = NULL;
126	nv_file_private_t *nvfp = NV_GET_FILE_PRIVATE(NV_VMA_FILE(vma));	126	nv_file_private_t *nvfp = NV_GET_FILE_PRIVATE(NV_VMA_FILE(vma));
127	nv_state_t *nv = NV_STATE_PTR(nvfp->nvptr);	127	nv_state_t *nv = NV_STATE_PTR(nvfp->nvptr);
128	NvU32 pageIndex, pageOffset;	128	NvU32 pageIndex, pageOffset;
129	void *kernel_mapping;	129	void *kernel_mapping;

Lines 95-134 Link Here

(-)kernel_/nvidia/nv-modeset-interface.c (-7 / +6 lines)
95	count++;	95	count++;
96	}	96	}
97		97
98	UNLOCK_NV_LINUX_DEVICES();	98	UNLOCK_NV_LINUX_DEVICES();
99		99
100	return count;	100	return count;
101	}	101	}
102		102
103	NV_STATUS nvidia_get_rm_ops(nvidia_modeset_rm_ops_t *rm_ops)	103	NV_STATUS nvidia_get_rm_ops(const nvidia_modeset_rm_ops_t rm_ops, const char version_string)
104	{	104	{
105	const nvidia_modeset_rm_ops_t local_rm_ops = {	105	static nvidia_modeset_rm_ops_t local_rm_ops = {
106	.version_string = NV_VERSION_STRING,
107	.system_info = {	106	.system_info = {
108	.allow_write_combining = NV_FALSE,	107	.allow_write_combining = NV_FALSE,
109	},	108	},
110	.alloc_stack = nvidia_modeset_rm_ops_alloc_stack,	109	.alloc_stack = nvidia_modeset_rm_ops_alloc_stack,
111	.free_stack = nvidia_modeset_rm_ops_free_stack,	110	.free_stack = nvidia_modeset_rm_ops_free_stack,
112	.enumerate_gpus = nvidia_modeset_enumerate_gpus,	111	.enumerate_gpus = nvidia_modeset_enumerate_gpus,
113	.open_gpu = nvidia_dev_get,	112	.open_gpu = nvidia_dev_get,
114	.close_gpu = nvidia_dev_put,	113	.close_gpu = nvidia_dev_put,
115	.op = rm_kernel_rmapi_op, /* provided by nv-kernel.o */	114	.op = rm_kernel_rmapi_op, /* provided by nv-kernel.o */
116	.set_callbacks = nvidia_modeset_set_callbacks,	115	.set_callbacks = nvidia_modeset_set_callbacks,
117	};	116	};
118		117
119	if (strcmp(rm_ops->version_string, NV_VERSION_STRING) != 0)	118	if (strcmp(*version_string, NV_VERSION_STRING) != 0)
120	{	119	{
121	rm_ops->version_string = NV_VERSION_STRING;	120	*version_string = NV_VERSION_STRING;
122	return NV_ERR_GENERIC;	121	return NV_ERR_GENERIC;
123	}	122	}
124		123
125	*rm_ops = local_rm_ops;	124	rm_ops = (const nvidia_modeset_rm_ops_t ) &local_rm_ops;
126		125
127	if (NV_ALLOW_WRITE_COMBINING(NV_MEMORY_TYPE_FRAMEBUFFER)) {	126	if (NV_ALLOW_WRITE_COMBINING(NV_MEMORY_TYPE_FRAMEBUFFER)) {
128	rm_ops->system_info.allow_write_combining = NV_TRUE;	127	local_rm_ops.system_info.allow_write_combining = NV_TRUE;
129	}	128	}
130		129
131	return NV_OK;	130	return NV_OK;
132	}	131	}
133		132
134	EXPORT_SYMBOL(nvidia_get_rm_ops);	133	EXPORT_SYMBOL(nvidia_get_rm_ops);

Lines 1307-1322 Link Here

(-)kernel_/common/inc/nv-linux.h (+3 lines)
1307	#else	1307	#else
1308	#define NV_KMEM_CACHE_CREATE_FULL(name, size, align, flags, ctor) \	1308	#define NV_KMEM_CACHE_CREATE_FULL(name, size, align, flags, ctor) \
1309	kmem_cache_create(name, size, align, flags, ctor, NULL)	1309	kmem_cache_create(name, size, align, flags, ctor, NULL)
1310	#endif	1310	#endif
1311		1311
1312	#define NV_KMEM_CACHE_CREATE(name, type) \	1312	#define NV_KMEM_CACHE_CREATE(name, type) \
1313	NV_KMEM_CACHE_CREATE_FULL(name, sizeof(type), 0, 0, NULL)	1313	NV_KMEM_CACHE_CREATE_FULL(name, sizeof(type), 0, 0, NULL)
1314		1314
		1315	#define NV_KMEM_CACHE_CREATE_USERCOPY(name, type) \
		1316	NV_KMEM_CACHE_CREATE_FULL(name, sizeof(type), 0, SLAB_USERCOPY, NULL)
		1317
1315	#define NV_KMEM_CACHE_DESTROY(kmem_cache) \	1318	#define NV_KMEM_CACHE_DESTROY(kmem_cache) \
1316	kmem_cache_destroy(kmem_cache)	1319	kmem_cache_destroy(kmem_cache)
1317		1320
1318	#define NV_KMEM_CACHE_ALLOC(kmem_cache) \	1321	#define NV_KMEM_CACHE_ALLOC(kmem_cache) \
1319	kmem_cache_alloc(kmem_cache, GFP_KERNEL)	1322	kmem_cache_alloc(kmem_cache, GFP_KERNEL)
1320	#define NV_KMEM_CACHE_FREE(ptr, kmem_cache) \	1323	#define NV_KMEM_CACHE_FREE(ptr, kmem_cache) \
1321	kmem_cache_free(kmem_cache, ptr)	1324	kmem_cache_free(kmem_cache, ptr)
1322		1325

Lines 67-83 Link Here

(-)kernel_/common/inc/nv-modeset-interface.h (-2 / +2 lines)
67	typedef struct {	67	typedef struct {
68	/*	68	/*
69	* The nvidia-modeset kernel module should assign version_string	69	* The nvidia-modeset kernel module should assign version_string
70	* before passing the structure to the nvidia kernel module, so	70	* before passing the structure to the nvidia kernel module, so
71	* that a version match can be confirmed: it is not supported to	71	* that a version match can be confirmed: it is not supported to
72	* mix nvidia and nvidia-modeset kernel modules from different	72	* mix nvidia and nvidia-modeset kernel modules from different
73	* releases.	73	* releases.
74	*/	74	*/
75	const char *version_string;	75	// const char *version_string;
76		76
77	/*	77	/*
78	* Return system information.	78	* Return system information.
79	*/	79	*/
80	struct {	80	struct {
81	/* Availability of write combining support for video memory */	81	/* Availability of write combining support for video memory */
82	NvBool allow_write_combining;	82	NvBool allow_write_combining;
83	} system_info;	83	} system_info;
Lines 112-122 Link Here
112		112
113	void NV_MODESET_INTERFACE_API_CALL (*op)	113	void NV_MODESET_INTERFACE_API_CALL (*op)
114	(nvidia_modeset_stack_ptr sp, void *ops_cmd);	114	(nvidia_modeset_stack_ptr sp, void *ops_cmd);
115		115
116	int (set_callbacks)(const nvidia_modeset_callbacks_t cb);	116	int (set_callbacks)(const nvidia_modeset_callbacks_t cb);
117		117
118	} nvidia_modeset_rm_ops_t;	118	} nvidia_modeset_rm_ops_t;
119		119
120	NV_STATUS nvidia_get_rm_ops(nvidia_modeset_rm_ops_t *rm_ops);	120	NV_STATUS nvidia_get_rm_ops(const nvidia_modeset_rm_ops_t rm_ops, const char version_string);
121		121
122	#endif /* _NV_MODESET_INTERFACE_H_ */	122	#endif /* _NV_MODESET_INTERFACE_H_ */

Lines 29-42 Link Here

(-)kernel_/common/inc/nv-register-module.h (-1 / +1 lines)
29		29
30	/* file operations */	30	/* file operations */
31	int (open)(struct inode , struct file *filp);	31	int (open)(struct inode , struct file *filp);
32	int (close)(struct inode , struct file *filp);	32	int (close)(struct inode , struct file *filp);
33	int (mmap)(struct file filp, struct vm_area_struct *vma);	33	int (mmap)(struct file filp, struct vm_area_struct *vma);
34	int (ioctl)(struct inode , struct file * file, unsigned int cmd, unsigned long arg);	34	int (ioctl)(struct inode , struct file * file, unsigned int cmd, unsigned long arg);
35	unsigned int (poll)(struct file file, poll_table *wait);	35	unsigned int (poll)(struct file file, poll_table *wait);
36		36
37	} nvidia_module_t;	37	} __do_const nvidia_module_t;
38		38
39	int nvidia_register_module(nvidia_module_t *);	39	int nvidia_register_module(nvidia_module_t *);
40	int nvidia_unregister_module(nvidia_module_t *);	40	int nvidia_unregister_module(nvidia_module_t *);
41		41
42	#endif	42	#endif

Lines 15-32 Link Here

(-)kernel_/nvidia/nv-chrdev.c (-2 lines)
15	#include "nv-linux.h"	15	#include "nv-linux.h"
16	#include "nv-frontend.h"	16	#include "nv-frontend.h"
17	#include "nv-instance.h"	17	#include "nv-instance.h"
18		18
19	int nv_register_chrdev(void *param)	19	int nv_register_chrdev(void *param)
20	{	20	{
21	nvidia_module_t module = (nvidia_module_t )param;	21	nvidia_module_t module = (nvidia_module_t )param;
22		22
23	module->instance = nv_module_instance;
24
25	return (nvidia_register_module(module));	23	return (nvidia_register_module(module));
26	}	24	}
27		25
28	void nv_unregister_chrdev(void *param)	26	void nv_unregister_chrdev(void *param)
29	{	27	{
30	nvidia_module_t module = (nvidia_module_t )param;	28	nvidia_module_t module = (nvidia_module_t )param;
31		29
32	nvidia_unregister_module(module);	30	nvidia_unregister_module(module);

Lines 46-61 Link Here

(-)kernel_/nvidia/nv-instance.c (+1 lines)
46	.err_handler = &nv_pci_error_handlers,	46	.err_handler = &nv_pci_error_handlers,
47	#endif	47	#endif
48	};	48	};
49		49
50	/* character device entry points*/	50	/* character device entry points*/
51	nvidia_module_t nv_fops = {	51	nvidia_module_t nv_fops = {
52	.owner = THIS_MODULE,	52	.owner = THIS_MODULE,
53	.module_name = MODULE_NAME,	53	.module_name = MODULE_NAME,
		54	.instance = MODULE_INSTANCE_NUMBER,
54	.open = nvidia_open,	55	.open = nvidia_open,
55	.close = nvidia_close,	56	.close = nvidia_close,
56	.ioctl = nvidia_ioctl,	57	.ioctl = nvidia_ioctl,
57	.mmap = nvidia_mmap,	58	.mmap = nvidia_mmap,
58	.poll = nvidia_poll,	59	.poll = nvidia_poll,
59	};	60	};
60		61
61	/*	62	/*

Lines 607-623 Link Here

(-)kernel_/nvidia-drm/nvidia-drm-drv.c (-1 / +3 lines)
607	DRM_IOCTL_DEF_DRV(NVIDIA_GEM_PRIME_FENCE_FORCE_SIGNAL,	607	DRM_IOCTL_DEF_DRV(NVIDIA_GEM_PRIME_FENCE_FORCE_SIGNAL,
608	nvidia_drm_gem_prime_fence_force_signal,	608	nvidia_drm_gem_prime_fence_force_signal,
609	DRM_CONTROL_ALLOW\|DRM_RENDER_ALLOW\|DRM_UNLOCKED),	609	DRM_CONTROL_ALLOW\|DRM_RENDER_ALLOW\|DRM_UNLOCKED),
610	DRM_IOCTL_DEF_DRV(NVIDIA_GEM_PRIME_FENCE_FINI,	610	DRM_IOCTL_DEF_DRV(NVIDIA_GEM_PRIME_FENCE_FINI,
611	nvidia_drm_gem_prime_fence_fini,	611	nvidia_drm_gem_prime_fence_fini,
612	DRM_CONTROL_ALLOW\|DRM_RENDER_ALLOW\|DRM_UNLOCKED),	612	DRM_CONTROL_ALLOW\|DRM_RENDER_ALLOW\|DRM_UNLOCKED),
613	};	613	};
614		614
615	static struct drm_driver nv_drm_driver = {	615	static drm_driver_no_const nv_drm_driver __read_only = {
616		616
617	.driver_features = DRIVER_GEM \| DRIVER_PRIME \| DRIVER_RENDER,	617	.driver_features = DRIVER_GEM \| DRIVER_PRIME \| DRIVER_RENDER,
618		618
619	.gem_free_object = nvidia_drm_gem_free,	619	.gem_free_object = nvidia_drm_gem_free,
620		620
621	.ioctls = nv_drm_ioctls,	621	.ioctls = nv_drm_ioctls,
622	.num_ioctls = ARRAY_SIZE(nv_drm_ioctls),	622	.num_ioctls = ARRAY_SIZE(nv_drm_ioctls),
623		623
Lines 668-691 Link Here
668	{	668	{
669	#if defined(NV_DRM_ATOMIC_MODESET_AVAILABLE)	669	#if defined(NV_DRM_ATOMIC_MODESET_AVAILABLE)
670		670
671	if (!nvidia_drm_modeset_module_param)	671	if (!nvidia_drm_modeset_module_param)
672	{	672	{
673	return;	673	return;
674	}	674	}
675		675
		676	pax_open_kernel();
676	nv_drm_driver.driver_features \|= DRIVER_MODESET \| DRIVER_ATOMIC;	677	nv_drm_driver.driver_features \|= DRIVER_MODESET \| DRIVER_ATOMIC;
677		678
678	nv_drm_driver.master_set = nvidia_drm_master_set;	679	nv_drm_driver.master_set = nvidia_drm_master_set;
679	nv_drm_driver.master_drop = nvidia_drm_master_drop;	680	nv_drm_driver.master_drop = nvidia_drm_master_drop;
680		681
681	nv_drm_driver.dumb_create = nvidia_drm_dumb_create;	682	nv_drm_driver.dumb_create = nvidia_drm_dumb_create;
682	nv_drm_driver.dumb_map_offset = nvidia_drm_dumb_map_offset;	683	nv_drm_driver.dumb_map_offset = nvidia_drm_dumb_map_offset;
683	nv_drm_driver.dumb_destroy = drm_gem_dumb_destroy;	684	nv_drm_driver.dumb_destroy = drm_gem_dumb_destroy;
		685	pax_close_kernel();
684		686
685	#endif /* NV_DRM_ATOMIC_MODESET_AVAILABLE */	687	#endif /* NV_DRM_ATOMIC_MODESET_AVAILABLE */
686	}	688	}
687		689
688		690
689		691
690	/*	692	/*
691	* Helper function for allocate/register DRM device for given NVIDIA GPU ID.	693	* Helper function for allocate/register DRM device for given NVIDIA GPU ID.

Lines 317-379 Link Here

(-)kernel_/nvidia-modeset/nvidia-modeset-linux.c (-20 / +20 lines)
317	up(&nvkms_lock);	317	up(&nvkms_lock);
318	}	318	}
319		319
320		320
321	/*************************************************************************	321	/*************************************************************************
322	* Interface with resman.	322	* Interface with resman.
323	*************************************************************************/	323	*************************************************************************/
324		324
325	static nvidia_modeset_rm_ops_t __rm_ops = { 0 };	325	static const nvidia_modeset_rm_ops_t *__rm_ops;
		326
326	static nvidia_modeset_callbacks_t nvkms_rm_callbacks = {	327	static nvidia_modeset_callbacks_t nvkms_rm_callbacks = {
327	nvkms_suspend,	328	.suspend = nvkms_suspend,
328	nvkms_resume	329	.resume = nvkms_resume
329	};	330	};
330		331
331	static int nvkms_alloc_rm(void)	332	static int nvkms_alloc_rm(void)
332	{	333	{
333	NV_STATUS nvstatus;	334	NV_STATUS nvstatus;
334	int ret;	335	int ret;
		336	const char * version_string = NV_VERSION_STRING;
335		337
336	__rm_ops.version_string = NV_VERSION_STRING;	338	nvstatus = nvidia_get_rm_ops(&__rm_ops, &version_string);
337
338	nvstatus = nvidia_get_rm_ops(&__rm_ops);
339		339
340	if (nvstatus != NV_OK) {	340	if (nvstatus != NV_OK) {
341	printk(KERN_ERR NVKMS_LOG_PREFIX "Version mismatch: "	341	printk(KERN_ERR NVKMS_LOG_PREFIX "Version mismatch: "
342	"nvidia.ko(%s) nvidia-modeset.ko(%s)\n",	342	"nvidia.ko(%s) nvidia-modeset.ko(%s)\n",
343	__rm_ops.version_string, NV_VERSION_STRING);	343	version_string, NV_VERSION_STRING);
344	return -EINVAL;	344	return -EINVAL;
345	}	345	}
346		346
347	ret = __rm_ops.set_callbacks(&nvkms_rm_callbacks);	347	ret = __rm_ops->set_callbacks(&nvkms_rm_callbacks);
348	if (ret < 0) {	348	if (ret < 0) {
349	printk(KERN_ERR NVKMS_LOG_PREFIX "Failed to register callbacks\n");	349	printk(KERN_ERR NVKMS_LOG_PREFIX "Failed to register callbacks\n");
350	return ret;	350	return ret;
351	}	351	}
352		352
353	return 0;	353	return 0;
354	}	354	}
355		355
356	static void nvkms_free_rm(void)	356	static void nvkms_free_rm(void)
357	{	357	{
358	__rm_ops.set_callbacks(NULL);	358	__rm_ops->set_callbacks(NULL);
359	}	359	}
360		360
361	void NVKMS_API_CALL nvkms_call_rm(void *ops)	361	void NVKMS_API_CALL nvkms_call_rm(void *ops)
362	{	362	{
363	nvidia_modeset_stack_ptr stack = NULL;	363	nvidia_modeset_stack_ptr stack = NULL;
364		364
365	if (__rm_ops.alloc_stack(&stack) != 0) {	365	if (__rm_ops->alloc_stack(&stack) != 0) {
366	return;	366	return;
367	}	367	}
368		368
369	__rm_ops.op(stack, ops);	369	__rm_ops->op(stack, ops);
370		370
371	__rm_ops.free_stack(stack);	371	__rm_ops->free_stack(stack);
372	}	372	}
373		373
374	/*************************************************************************	374	/*************************************************************************
375	* ref_ptr implementation.	375	* ref_ptr implementation.
376	*************************************************************************/	376	*************************************************************************/
377		377
378	struct nvkms_ref_ptr {	378	struct nvkms_ref_ptr {
379	struct kref refcnt;	379	struct kref refcnt;
Lines 685-732 Link Here
685	return data;	685	return data;
686	}	686	}
687		687
688	NvBool NVKMS_API_CALL nvkms_open_gpu(NvU32 gpuId)	688	NvBool NVKMS_API_CALL nvkms_open_gpu(NvU32 gpuId)
689	{	689	{
690	nvidia_modeset_stack_ptr stack = NULL;	690	nvidia_modeset_stack_ptr stack = NULL;
691	NvBool ret;	691	NvBool ret;
692		692
693	if (__rm_ops.alloc_stack(&stack) != 0) {	693	if (__rm_ops->alloc_stack(&stack) != 0) {
694	return NV_FALSE;	694	return NV_FALSE;
695	}	695	}
696		696
697	ret = __rm_ops.open_gpu(gpuId, stack) == 0;	697	ret = __rm_ops->open_gpu(gpuId, stack) == 0;
698		698
699	__rm_ops.free_stack(stack);	699	__rm_ops->free_stack(stack);
700		700
701	return ret;	701	return ret;
702	}	702	}
703		703
704	void NVKMS_API_CALL nvkms_close_gpu(NvU32 gpuId)	704	void NVKMS_API_CALL nvkms_close_gpu(NvU32 gpuId)
705	{	705	{
706	nvidia_modeset_stack_ptr stack = NULL;	706	nvidia_modeset_stack_ptr stack = NULL;
707		707
708	if (__rm_ops.alloc_stack(&stack) != 0) {	708	if (__rm_ops->alloc_stack(&stack) != 0) {
709	return;	709	return;
710	}	710	}
711		711
712	__rm_ops.close_gpu(gpuId, stack);	712	__rm_ops->close_gpu(gpuId, stack);
713		713
714	__rm_ops.free_stack(stack);	714	__rm_ops->free_stack(stack);
715	}	715	}
716		716
717	NvU32 NVKMS_API_CALL nvkms_enumerate_gpus(nv_gpu_info_t *gpu_info)	717	NvU32 NVKMS_API_CALL nvkms_enumerate_gpus(nv_gpu_info_t *gpu_info)
718	{	718	{
719	return __rm_ops.enumerate_gpus(gpu_info);	719	return __rm_ops->enumerate_gpus(gpu_info);
720	}	720	}
721		721
722	NvBool NVKMS_API_CALL nvkms_allow_write_combining(void)	722	NvBool NVKMS_API_CALL nvkms_allow_write_combining(void)
723	{	723	{
724	return __rm_ops.system_info.allow_write_combining;	724	return __rm_ops->system_info.allow_write_combining;
725	}	725	}
726		726
727	/*************************************************************************	727	/*************************************************************************
728	* Common to both user-space and kapi NVKMS interfaces	728	* Common to both user-space and kapi NVKMS interfaces
729	*************************************************************************/	729	*************************************************************************/
730		730
731	static void nvkms_kapi_event_work_queue_callback(NVKMS_WORK_FUNC_ARG_T *arg)	731	static void nvkms_kapi_event_work_queue_callback(NVKMS_WORK_FUNC_ARG_T *arg)
732	{	732	{

Lines 30-56 Link Here

(-)kernel_/nvidia-uvm/uvm8_global.c (-5 / +5 lines)
30	#include "uvm8_thread_context.h"	30	#include "uvm8_thread_context.h"
31	#include "uvm8_va_range.h"	31	#include "uvm8_va_range.h"
32	#include "uvm8_kvmalloc.h"	32	#include "uvm8_kvmalloc.h"
33	#include "uvm8_mmu.h"	33	#include "uvm8_mmu.h"
34	#include "uvm8_perf_heuristics.h"	34	#include "uvm8_perf_heuristics.h"
35	#include "nv_uvm_interface.h"	35	#include "nv_uvm_interface.h"
36		36
37	uvm_global_t g_uvm_global;	37	uvm_global_t g_uvm_global;
38	static struct UvmOpsUvmEvents g_exported_uvm8_ops;	38	static struct UvmOpsUvmEvents g_exported_uvm8_ops = {
		39	.startDevice = NULL,
		40	.stopDevice = NULL,
		41	.isrTopHalf = uvm8_isr_top_half,
		42	};
39	static bool g_ops_registered = false;	43	static bool g_ops_registered = false;
40		44
41	static NV_STATUS uvm8_register_callbacks(void)	45	static NV_STATUS uvm8_register_callbacks(void)
42	{	46	{
43	NV_STATUS status = NV_OK;	47	NV_STATUS status = NV_OK;
44		48
45	g_exported_uvm8_ops.startDevice = NULL;
46	g_exported_uvm8_ops.stopDevice = NULL;
47	g_exported_uvm8_ops.isrTopHalf = uvm8_isr_top_half;
48
49	// Register the UVM callbacks with the main GPU driver:	49	// Register the UVM callbacks with the main GPU driver:
50	status = uvm_rm_locked_call(nvUvmInterfaceRegisterUvmCallbacks(&g_exported_uvm8_ops));	50	status = uvm_rm_locked_call(nvUvmInterfaceRegisterUvmCallbacks(&g_exported_uvm8_ops));
51	if (status != NV_OK)	51	if (status != NV_OK)
52	return status;	52	return status;
53		53
54	g_ops_registered = true;	54	g_ops_registered = true;
55	return NV_OK;	55	return NV_OK;
56	}	56	}

Lines 347-379 Link Here

(-)kernel_/nvidia-uvm/uvm8_gpu_semaphore.c (-2 / +2 lines)
347	{	347	{
348	NvU32 index = get_index(semaphore);	348	NvU32 index = get_index(semaphore);
349	NvU64 base_va = uvm_rm_mem_get_gpu_va(semaphore->page->memory, gpu);	349	NvU64 base_va = uvm_rm_mem_get_gpu_va(semaphore->page->memory, gpu);
350	return base_va + UVM_SEMAPHORE_SIZE * index;	350	return base_va + UVM_SEMAPHORE_SIZE * index;
351	}	351	}
352		352
353	NvU32 uvm_gpu_semaphore_get_payload(uvm_gpu_semaphore_t *semaphore)	353	NvU32 uvm_gpu_semaphore_get_payload(uvm_gpu_semaphore_t *semaphore)
354	{	354	{
355	return ACCESS_ONCE(*semaphore->payload);	355	return ACCESS_ONCE_RW(*semaphore->payload);
356	}	356	}
357		357
358	void uvm_gpu_semaphore_set_payload(uvm_gpu_semaphore_t *semaphore, NvU32 payload)	358	void uvm_gpu_semaphore_set_payload(uvm_gpu_semaphore_t *semaphore, NvU32 payload)
359	{	359	{
360	// Provide a guarantee that all memory accesses prior to setting the payload	360	// Provide a guarantee that all memory accesses prior to setting the payload
361	// won't be moved past it.	361	// won't be moved past it.
362	// Use a big hammer mb() as set_payload() is not used in any performance path	362	// Use a big hammer mb() as set_payload() is not used in any performance path
363	// today.	363	// today.
364	// This could likely be optimized to be either an smp_store_release() or use	364	// This could likely be optimized to be either an smp_store_release() or use
365	// an smp_mb__before_atomic() barrier. The former is a recent addition to	365	// an smp_mb__before_atomic() barrier. The former is a recent addition to
366	// kernel though, and it's not clear whether combining the latter with a	366	// kernel though, and it's not clear whether combining the latter with a
367	// regular 32bit store is well defined in all cases. Both also seem to risk	367	// regular 32bit store is well defined in all cases. Both also seem to risk
368	// being optimized out on non-SMP configs (we need them for interacting with	368	// being optimized out on non-SMP configs (we need them for interacting with
369	// the GPU correctly even on non-SMP).	369	// the GPU correctly even on non-SMP).
370	mb();	370	mb();
371	ACCESS_ONCE(*semaphore->payload) = payload;	371	ACCESS_ONCE_RW(*semaphore->payload) = payload;
372	}	372	}
373		373
374	// This function is intended to catch channels which have been left dangling in	374	// This function is intended to catch channels which have been left dangling in
375	// trackers after their owning GPUs have been destroyed.	375	// trackers after their owning GPUs have been destroyed.
376	static bool tracking_semaphore_check_gpu(uvm_gpu_tracking_semaphore_t *tracking_sem)	376	static bool tracking_semaphore_check_gpu(uvm_gpu_tracking_semaphore_t *tracking_sem)
377	{	377	{
378	uvm_gpu_t *gpu = tracking_sem->semaphore.page->pool->gpu;	378	uvm_gpu_t *gpu = tracking_sem->semaphore.page->pool->gpu;
379	uvm_gpu_t *table_gpu;	379	uvm_gpu_t *table_gpu;

Lines 311-327 Link Here

(-)kernel_/nvidia-uvm/uvm8_hal.h (-1 / +1 lines)
311	uvm_ce_hal_t ce_ops;	311	uvm_ce_hal_t ce_ops;
312		312
313	// arch_ops: id is an architecture	313	// arch_ops: id is an architecture
314	uvm_arch_hal_t arch_ops;	314	uvm_arch_hal_t arch_ops;
315		315
316	// fault_buffer_ops: id is a hardware class	316	// fault_buffer_ops: id is a hardware class
317	uvm_fault_buffer_hal_t fault_buffer_ops;	317	uvm_fault_buffer_hal_t fault_buffer_ops;
318	} u;	318	} u;
319	} uvm_hal_class_ops_t;	319	} __do_const uvm_hal_class_ops_t;
320		320
321	// When UVM next support is enabled support for future chips in the hal is	321	// When UVM next support is enabled support for future chips in the hal is
322	// enabled by providing additional hal table entries below.	322	// enabled by providing additional hal table entries below.
323	#if UVM_IS_NEXT()	323	#if UVM_IS_NEXT()
324	extern uvm_hal_class_ops_t *ce_table_next;	324	extern uvm_hal_class_ops_t *ce_table_next;
325	extern uvm_hal_class_ops_t *host_table_next;	325	extern uvm_hal_class_ops_t *host_table_next;
326	extern uvm_hal_class_ops_t *arch_table_next;	326	extern uvm_hal_class_ops_t *arch_table_next;
327	extern uvm_hal_class_ops_t *fault_buffer_table_next;	327	extern uvm_hal_class_ops_t *fault_buffer_table_next;

Lines 19-35 Link Here

(-)kernel_/nvidia-uvm/uvm8_mmu.h (-1 lines)
19	FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER	19	FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20	DEALINGS IN THE SOFTWARE.	20	DEALINGS IN THE SOFTWARE.
21		21
22	*******************************************************************************/	22	*******************************************************************************/
23		23
24	#ifndef __UVM8_MMU_H__	24	#ifndef __UVM8_MMU_H__
25	#define __UVM8_MMU_H__	25	#define __UVM8_MMU_H__
26		26
27	#include "uvm8_forward_decl.h"
28	#include "uvm8_hal_types.h"	27	#include "uvm8_hal_types.h"
29	#include "uvm8_pmm_gpu.h"	28	#include "uvm8_pmm_gpu.h"
30	#include "uvmtypes.h"	29	#include "uvmtypes.h"
31	#include "uvm_common.h"	30	#include "uvm_common.h"
32	#include "uvm8_tracker.h"	31	#include "uvm8_tracker.h"
33	#include "uvm8_test_ioctl.h"	32	#include "uvm8_test_ioctl.h"
34		33
35	// Used when the page size isn't known and should not matter	34	// Used when the page size isn't known and should not matter

Lines 46-62 Link Here

(-)kernel_/nvidia-uvm/uvm_common.c (-3 / +4 lines)
46	#include "uvm8_init.h"	46	#include "uvm8_init.h"
47	#include "uvm8_forward_decl.h"	47	#include "uvm8_forward_decl.h"
48		48
49	// TODO: Bug 1710855: Tweak this number through benchmarks	49	// TODO: Bug 1710855: Tweak this number through benchmarks
50	#define UVM_SPIN_LOOP_SCHEDULE_TIMEOUT_NS (10*1000ULL)	50	#define UVM_SPIN_LOOP_SCHEDULE_TIMEOUT_NS (10*1000ULL)
51	#define UVM_SPIN_LOOP_PRINT_TIMEOUT_SEC 30ULL	51	#define UVM_SPIN_LOOP_PRINT_TIMEOUT_SEC 30ULL
52		52
53	static dev_t g_uvmBaseDev;	53	static dev_t g_uvmBaseDev;
54	struct UvmOpsUvmEvents g_exportedUvmOps;
55		54
56	static char* uvm_driver_mode = "8";	55	static char* uvm_driver_mode = "8";
57		56
58	// UVM-Lite behavior is activated through insmod argument: (default is "8")	57	// UVM-Lite behavior is activated through insmod argument: (default is "8")
59	// insmod nvidia-uvm.ko uvm_driver_mode="lite"	58	// insmod nvidia-uvm.ko uvm_driver_mode="lite"
60	//	59	//
61	// Similarly, the UVM-8 production driver can be activated with:	60	// Similarly, the UVM-8 production driver can be activated with:
62	// insmod nvidia-uvm.ko uvm_driver_mode="8"	61	// insmod nvidia-uvm.ko uvm_driver_mode="8"
Lines 173-190 Link Here
173		172
174	return nvStatus;	173	return nvStatus;
175	}	174	}
176		175
177	static NV_STATUS uvmSetupGpuProvider(void)	176	static NV_STATUS uvmSetupGpuProvider(void)
178	{	177	{
179	NV_STATUS status = NV_OK;	178	NV_STATUS status = NV_OK;
180		179
181	g_exportedUvmOps.startDevice = uvm_gpu_event_start_device;	180	static struct UvmOpsUvmEvents g_exportedUvmOps = {
182	g_exportedUvmOps.stopDevice = uvm_gpu_event_stop_device;	181	.startDevice = uvm_gpu_event_start_device,
		182	.stopDevice = uvm_gpu_event_stop_device,
		183	};
183		184
184	// call RM to exchange the function pointers.	185	// call RM to exchange the function pointers.
185	status = nvUvmInterfaceRegisterUvmCallbacks(&g_exportedUvmOps);	186	status = nvUvmInterfaceRegisterUvmCallbacks(&g_exportedUvmOps);
186	return status;	187	return status;
187	}	188	}
188		189
189	static int uvm_not8_init(dev_t uvmBaseDev)	190	static int uvm_not8_init(dev_t uvmBaseDev)
190	{	191	{

Lines 25-40 Link Here

(-)kernel_/nvidia-uvm/uvm_full_fault_buffer.h (-1 / +2 lines)
25	// uvm_full_fault_buffer.h	25	// uvm_full_fault_buffer.h
26	//	26	//
27	// This file contains structures and function declarations to read or update	27	// This file contains structures and function declarations to read or update
28	// the fault buffer/related registers and read/mask/unmask the fault intr.	28	// the fault buffer/related registers and read/mask/unmask the fault intr.
29	//	29	//
30	#ifndef _UVM_FULL_FAULT_BUFFER_H_	30	#ifndef _UVM_FULL_FAULT_BUFFER_H_
31	#define _UVM_FULL_FAULT_BUFFER_H_	31	#define _UVM_FULL_FAULT_BUFFER_H_
32		32
		33	#include <linux/compiler.h>
33	#include "uvmtypes.h"	34	#include "uvmtypes.h"
34		35
35	#define MAXWELL_FAULT_BUFFER_A (0xb069)	36	#define MAXWELL_FAULT_BUFFER_A (0xb069)
36	#define MEM_RD32(a) ((const volatile NvU32 )(a))	37	#define MEM_RD32(a) ((const volatile NvU32 )(a))
37	#define MEM_WR32(a, d) do { (volatile NvU32 )(a) = (d); } while (0)	38	#define MEM_WR32(a, d) do { (volatile NvU32 )(a) = (d); } while (0)
38	typedef enum	39	typedef enum
39	{	40	{
40	UvmReplayType_none,	41	UvmReplayType_none,
Lines 298-314 Link Here
298	NvUvmSetReplayParamsReg_t setReplayParamsReg;	299	NvUvmSetReplayParamsReg_t setReplayParamsReg;
299	NvUvmGetFaultPacketSize_t getFaultPacketSize;	300	NvUvmGetFaultPacketSize_t getFaultPacketSize;
300	NvUvmWriteFaultBufferPacket_t writeFaultBufferPacket;	301	NvUvmWriteFaultBufferPacket_t writeFaultBufferPacket;
301	NvUvmIsFaultInterruptPending_t isFaultIntrPending;	302	NvUvmIsFaultInterruptPending_t isFaultIntrPending;
302	NvUvmSetFaultIntrBit_t setFaultIntrBit;	303	NvUvmSetFaultIntrBit_t setFaultIntrBit;
303	NvUvmControlPrefetch_t controlPrefetch;	304	NvUvmControlPrefetch_t controlPrefetch;
304	NvUvmTestFaultBufferOverflow_t testFaultBufferOverflow;	305	NvUvmTestFaultBufferOverflow_t testFaultBufferOverflow;
305	NvUvmClearFaultBufferOverflow_t clearFaultBufferOverflow;	306	NvUvmClearFaultBufferOverflow_t clearFaultBufferOverflow;
306	} UvmFaultBufferOps;	307	} __no_const UvmFaultBufferOps;
307		308
308	/******************************************************************************	309	/******************************************************************************
309	uvmfull_fault_buffer_init	310	uvmfull_fault_buffer_init
310	Initialze fault buffer management related function pointers for a	311	Initialze fault buffer management related function pointers for a
311	fault class	312	fault class
312		313
313	Arguments:	314	Arguments:
314	faultBufferClass: (INPUT)	315	faultBufferClass: (INPUT)

Lines 415-431 Link Here

(-)kernel_/nvidia-uvm/uvm_linux.h (-1 / +1 lines)
415		415
416	// Added in 2.6.24	416	// Added in 2.6.24
417	#ifndef ACCESS_ONCE	417	#ifndef ACCESS_ONCE
418	#define ACCESS_ONCE(x) ((volatile typeof(x) )&(x))	418	#define ACCESS_ONCE(x) ((volatile typeof(x) )&(x))
419	#endif	419	#endif
420		420
421	// WRITE_ONCE/READ_ONCE have incompatible definitions across versions, which produces warnings.	421	// WRITE_ONCE/READ_ONCE have incompatible definitions across versions, which produces warnings.
422	// Therefore, we define our own macros	422	// Therefore, we define our own macros
423	#define UVM_WRITE_ONCE(x, val) (ACCESS_ONCE(x) = (val))	423	#define UVM_WRITE_ONCE(x, val) (ACCESS_ONCE_RW(x) = (val))
424	#define UVM_READ_ONCE(x) ACCESS_ONCE(x)	424	#define UVM_READ_ONCE(x) ACCESS_ONCE(x)
425		425
426	// Added in 3.11	426	// Added in 3.11
427	#ifndef PAGE_ALIGNED	427	#ifndef PAGE_ALIGNED
428	#define PAGE_ALIGNED(addr) (((addr) & (PAGE_SIZE - 1)) == 0)	428	#define PAGE_ALIGNED(addr) (((addr) & (PAGE_SIZE - 1)) == 0)
429	#endif	429	#endif
430		430
431	// Added in 2.6.37 via commit e1ca7788dec6773b1a2bce51b7141948f2b8bccf	431	// Added in 2.6.37 via commit e1ca7788dec6773b1a2bce51b7141948f2b8bccf