1 int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
 2             unsigned long pfn, unsigned long size, pgprot_t prot)
 3 {
 4     pgd_t *pgd;
 5     unsigned long next;
 6     unsigned long end = addr + PAGE_ALIGN(size);
 7     struct mm_struct *mm = vma->vm_mm;
 8     unsigned long remap_pfn = pfn;
 9     int err;
10     
11     /*
12      * Physically remapped pages are special. Tell the
13      * rest of the world about it:
14      *   VM_IO tells people not to look at these pages
15      *    (accesses can have side effects).
16      *   VM_PFNMAP tells the core MM that the base pages are just
17      *    raw PFN mappings, and do not have a "struct page" associated
18      *    with them.
19      *   VM_DONTEXPAND
20      *      Disable vma merging and expanding with mremap().
21      *   VM_DONTDUMP
22      *      Omit vma from core dump, even when VM_IO turned off.
23      *
24      * There‘s a horrible special case to handle copy-on-write
25      * behaviour that some programs depend on. We mark the "original"
26      * un-COW‘ed pages by matching them up with "vma->vm_pgoff".
27      * See vm_normal_page() for details.
28      */
29     vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
30 
31     BUG_ON(addr >= end);
32     pfn -= addr >> PAGE_SHIFT;
33     pgd = pgd_offset(mm, addr);
34     flush_cache_range(vma, addr, end);
35     do {
36         next = pgd_addr_end(addr, end);
37         err = remap_pud_range(mm, pgd, addr, next,
38                 pfn + (addr >> PAGE_SHIFT), prot);
39         if (err)
40             break;
41     } while (pgd++, addr = next, addr != end);
42 
43     return err;
44 }

 1 static inline int remap_pud_range(struct mm_struct *mm, pgd_t *pgd,
 2             unsigned long addr, unsigned long end,
 3             unsigned long pfn, pgprot_t prot)
 4 {
 5     pud_t *pud;
 6     unsigned long next;
 7 
 8     pfn -= addr >> PAGE_SHIFT;
 9     pud = pud_alloc(mm, pgd, addr);
10     if (!pud)
11         return -ENOMEM;
12     do {
13         next = pud_addr_end(addr, end);
14         if (remap_pmd_range(mm, pud, addr, next,
15                 pfn + (addr >> PAGE_SHIFT), prot))
16             return -ENOMEM;
17     } while (pud++, addr = next, addr != end);
18     return 0;
19 }

 1 static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud,
 2             unsigned long addr, unsigned long end,
 3             unsigned long pfn, pgprot_t prot)
 4 {
 5     pmd_t *pmd;
 6     unsigned long next;
 7 
 8     pfn -= addr >> PAGE_SHIFT;
 9     pmd = pmd_alloc(mm, pud, addr);
10     if (!pmd)
11         return -ENOMEM;
12     VM_BUG_ON(pmd_trans_huge(*pmd));
13     do {
14         next = pmd_addr_end(addr, end);
15         if (remap_pte_range(mm, pmd, addr, next,
16                 pfn + (addr >> PAGE_SHIFT), prot))
17             return -ENOMEM;
18     } while (pmd++, addr = next, addr != end);
19     return 0;
20 }

 1 static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd,
 2             unsigned long addr, unsigned long end,
 3             unsigned long pfn, pgprot_t prot)
 4 {
 5     pte_t *pte;
 6     spinlock_t *ptl;
 7 
 8     pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);
 9     if (!pte)
10         return -ENOMEM;
11     arch_enter_lazy_mmu_mode();
12     do {
13         BUG_ON(!pte_none(*pte));
14         set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot)));
15         pfn++;
16     } while (pte++, addr += PAGE_SIZE, addr != end);
17     arch_leave_lazy_mmu_mode();
18     pte_unmap_unlock(pte - 1, ptl);
19     return 0;
20 }

#define pte_alloc_map_lock(mm, pmd, address, ptlp)    \
    (pte_alloc(mm, pmd, address) ?                     NULL : pte_offset_map_lock(mm, pmd, address, ptlp))