#define SCCSID  "@(#)pgattach.c 6.6 92/03/27"

/*
 *      IBM/Microsoft Confidential
 *
 *      Copyright (c) IBM Corporation 1987, 1989
 *      Copyright (c) Microsoft Corporation 1987, 1989
 *
 *      All Rights Reserved
 */

#define FNO_NUM  FNO_PGATTACH_C
#include <fnomem.h>

#define INCL_ERRORS
#include <bseerr.h>
#include <osconst.h>
#include <ostypes.h>
#include <osmac.h>
#include <mi.h>
#include <dossymc.h>
#include <sigdef.h>
#include <except.h>
#include <smexport.h>
#include <pgexport.h>
#include <vmexport.h>
#include <tkexport.h>
#include <infoseg.h>
#include <curdir.h>
#include <dirent.h>
#include <find.h>
#include <screen.h>
#include <sem.h>
#include <tkptda.h>
#define INCL_TYPES
#include <basedef.h>
#include <mvdmexp.h>
#include <pglocal.h>
#include <stdlib.h>
#include <kutil.h>
#include <string.h>
#include <vmowner.h>

extern ulong_t CpuStepping;

#pragma code_seg("SWAP32_7","CODE")

/***EP  PGAttach - create page table entry aliases
 *
 *      PGAttach points a range of ptes (the new range) to the same physical
 *      storage as another range (the old range).  The ptes in the new
 *      range must be invalid and unallocated (zero) (unless the PG_REMAP flag
 *      is specified -- see below).  If a page in the old range
 *      is valid, the corresponding page in the new range will be
 *      attached to it.  If a page in the old range is unallocated,
 *      the correspondig page in the new range will remain invalid and
 *      no error will be returned.
 *
 *      Writable attachments to discardable pages cause the
 *      old and new pages to become swappable.  If the old page is discardable
 *      and the PG_PRIV flag is specified, the old page will be split
 *      from any other attachments that might have been sharing the page.
 *
 *      The PG_ flags are used to control the access permissions of
 *      the new pages.  If no PG_ flags are specified, the permissions
 *      will be inheritted from the old pages.
 *      The Guard attribute will not be inheritted, but can be specified
 *      explicitly with the PG_GUARD flag.
 *
 *      The PG_REMAP flag is used to allow the remapping of an already
 *      committed region -- in essence, an atomic free and attach.
 *
 *      The PG_SWITCHMAP flag is used in conjuction with the PG_REMAP flag
 *      but the SWITCHMAP flag causes the source and destination ptes tobe
 *      switched.  NOTE: the source and destination must map the same
 *      linear address region.
 *
 *      The caller must ensure that the old and new ranges do not overlap.
 *
 *      ENTRY   pagenew - destination virtual page number
 *              pptdanew - pointer to destination ptda
 *              pageold - source virtual page
 *              pptdaold - pointer to source ptda
 *              npages - number of pages
 *              flags - PG_* for permissions of new pages
 *                      PG_PRIV - make read/write attaches to read-only
 *                                      loader objects private
 *                      PG_REMAP - (see above)
 *                      PG_SWITCHMAP - (see above)
 *      EXIT    0 on success, else error code
 *
 *      setup permission masks
 *      call pgDriver2 with pgwAttach worker
 */
int
PGAttach(pagenew, pptdanew, pageold, pptdaold, npages, flags)
page_t pagenew;
struct ptda_s *pptdanew;
page_t pageold;
struct ptda_s *pptdaold;
int npages;
ulong_t flags;
{
    struct pginfo_s pgi[2];
    int err;

// GSR start
    /*
     *  If we're attaching something within the global shared region, just
     *   return NO_ERROR since were already using the same page tables
     */
    if (pagenew == pageold && pagenew < MAXGLOBALSHRPAGE && pagenew >= MINGLOBALSHRPAGE)
      return(0);
// GSR end

    /*
     *  If caller wants to set access, setup mask and access accordingly.
     *  Otherwise, set up to get all permissions from old range.
     *  New ptes will inherit those bits set in data2 and have set
     *  unconditionally those bits set in data1.
     */
    pgi[1].pi_data2 = ~(PT_NPGUARD | PT_RING3ACCESS | PT_RING3WRITE);
    pgi[1].pi_data1 = flags & (PG_U | PG_W | PG_GUARD);
    if (pgi[1].pi_data1 == 0) {
        pgi[1].pi_data2 = ~PT_NPGUARD;  /* don't inherit guard */
    }

    /*
     *  Setup rest of arguments for pgDriver2
     */
    pgi[0].pi_page = pagenew;
    pgi[0].pi_pptda = pptdanew;
    pgi[0].pi_flags = flags | PG_ALLOC;
    pgi[1].pi_page = pageold;
    pgi[1].pi_pptda = pptdaold;
    pgi[1].pi_flags = flags | PG_SKIPFREE;

    err = pgDriver2(SSToDS(&pgi), npages, pgwAttach, pgwFree);
//  if (pagenew != pageold) pgFlushTLB();
    pgFlushTLB();
    return(err);
}


/***LP  pgwAttach - PGAttach worker for pgDriver2
 *
 *      pgwAttach points a pte to the same physical storage as another.
 *
 *      See PGAttach() for a full description of behavior and flags.
 *
 *      ENTRY:  ppgi - pointer to an array of two pginfo_s structures.
 *                     The first pginfo structure describes the new range,
 *                     the second the old.
 *                     Special values:
 *                         ppgi[1].pi_data1 = set these bits in new pte
 *                         ppgi[1].pi_data2 = inherit these bits from old pte
 *
 *      if (old pte is resident)
 *          if (pt resident count is 0)
 *              call pgPinForcePresent with resident flag
 *         inc new pt resident count
 *      if (writeable attach to readonly discardable page)
 *          get a page of swappable overcommit
 *          free a page of discardable commit
 *          if (PG_PRIVATIZE)
 *              allocate new vp
 *              copy existing vp into new vp
 *              set vp ref count to 1
 *              attach vp to old pte
 *              make pginfo structure point to new vp
 *              clear old vp sem
 *          set vp sow bit
 *      inc new pt allocated count
 *      if (pte is present)
 *          if (guard page request)
 *              mark pte not present
 *              put vpi in frame field
 *              inc vp ref count
 *          else
 *              inc new pt present count
 *              if (page is not uvirt)
 *                  inc pf ref count
 *                  inc vp ref count
 *      else if (page is not uvirt)
 *          inc vp ref count
 *      copy old pte to new
 *      set new pte permission bits
 */
int
pgwAttach(ppgi)
register struct pginfo_s *ppgi;
{
    ulong_t err;
    struct vp_s *pvp;
    struct pf_s *ppf, *ppf1;
    int swappable,
        commited = 0,
        reloaded = 0;

    /*
     *  VM will always protect us from Attaching to a range we are already
     *  attached to.  BUGBUG - bs - I hope.
     */
    pgAssert(ppgi[0].pi_pte.pt_long == 0);

    /*
     *  If we are attaching to a resident page, increment the pt resident
     *  count for the new range.  If this is first resident page on the
     *  page table, get resident overcommit for the pt.
     */
    if (ppgi[1].pi_pte.pt_resident) {

        /*
         *  Don't allow resident guard pages.
         */
        if (ppgi[0].pi_flags & PG_GUARD) {
            return(ERROR_ACCESS_DENIED);
        }
        if (ppgi[0].pi_ppd->pd_pcresident[ppgi[0].pi_pdi] == 0) {
            if (err = pgPinForcePresent(ppgi[0].pi_pdi, ppgi[0].pi_ppd,
                                        PG_RESIDENT)) {
                return(err);
            }
        }
        ppgi[0].pi_ppd->pd_pcresident[ppgi[0].pi_pdi]++;

    /*
     *  If requested make this page private if it is not already private.
     */
    } else if ((ppgi[1].pi_flags & PG_PRIV) &&
              (!(ppgi[1].pi_pvp->vp_flags & VP_PRIVATIZED))) {

        pgAssert((ppgi[1].pi_flags & PG_W) &&
                 (ppgi[1].pi_pte.pt_ring3write == 0) &&
                 (ppgi[1].pi_pvp->vp_resident == 0));

        /*
         * Set swappable if the page is not discardable, or
         * the page is discardable, and swap-on-write, and has
         * been dirtied (swap-on-load).
         */
        swappable = ((ppgi[1].pi_pvp->vp_discardable == 0) ||
                    ((ppgi[1].pi_pvp->vp_discardable) &&
                    (ppgi[1].pi_pvp->vp_sow == 1) &&
                    (ppgi[1].pi_pvp->vp_flags & VP_DIRTY)));

        /*
         * Commit one page of swappable commit if the page
         * is currently being shared.
         */
        if (ppgi[1].pi_pvp->vp_refcount > 1) {

            if (err = pgCommit(1, PG_SWAPPABLE))
                return(err);

            commited++;
        }

        /*
         * When privatizing a page, break off a new copy of the
         * page. This is done so we always have a fresh copy of
         * the page to use when other processes attach to it,
         */
        if (err = pgGetPF(ppgi[0].pi_page, 0, SSToDS(&ppf1)))
            goto common_err1;

        ppf1->pf_busy = 1;
        ppf1->pf_pvp = NULL;
        ppf1->pf_refcount = 0;

        /*
         * Make sure that the orignal vp has a pf associated with it.
         */
        if (!ppgi[1].pi_pvp->vp_pf) {

            if (err = pgReloadFrame(ppgi[1].pi_page, ppgi[1].pi_pvp,
                                    ppgi[1].pi_pptda, 0, 0)) {
             common_err:
                ppf1->pf_busy = 0;
                pgFreePF(ppf1, pgPFToFrame(ppf1));

             common_err1:
                if (commited)
                    pgCommitFree(1, PG_SWAPPABLE);

                return(err);
            }
            reloaded++;
        }

        memcpy((void *)pgAliasTemp(pgPFToFrame(ppf1), 0),
               (void *)pgAliasTemp(ppgi[1].pi_pvp->vp_frame, 1),
               PAGESIZE);

        /*
         *  If the page is currently being shared, allocate a new page and
         *  copy the contents so we don't screw up others.
         */
        if (ppgi[1].pi_pvp->vp_refcount > 1) {

            pvp = pgAllocVP();
            pvp->vp_flags = VP_BUSY | VP_PRIVATIZED;
            pvp->vp_refcount = 1;
            pvp->vp_obpg = ppgi[1].pi_pvp->vp_obpg;
            pvp->vp_hob = ppgi[1].pi_pvp->vp_hob;
            pgAssert(ppgi[1].pi_pvp->vp_pf);
            pvp->vp_block = pgFrameToPF(ppgi[1].pi_pvp->vp_frame)->pf_block;

            /*
             *  If the old page is discardable, we can just mark the
             *  new one as SOW and reload it from the same place we do
             *  the others.
             */
            if (!swappable) {

                pvp->vp_flags |= (VP_SOW | VP_DISCARDABLE);

            /*
             *  The master page must currently be swappable.  So we must
             *  allocate a new physical page and copy the contents into
             *  it if the frame is around, otherwise we must swap in
             *  a copy (we must do this rather than have two vp's pointing
             *  the same DF because if one context dirtied the page, we
             *  would free the DF and loose the other context's data).
             */
            } else {
                /*
                 *  Make copy of existing pf for new page
                 */
                pgAssert(ppgi[1].pi_pvp->vp_pf);

                if (err = pgGetPF(ppgi[0].pi_page, 0, SSToDS(&ppf))) {
                  newpageerror:
                    pgFreeVP(pvp);
                    goto common_err;
                }
                ppf->pf_busy = 1;

                ppf->pf_block = pvp->vp_block;
                pvp->vp_frame = BLINK(ppf);

                ppf->pf_pvp = pvp;
                ppf->pf_refcount = 0;

                pvp->vp_pf = 1;

                memcpy((void *)pgAliasTemp(pvp->vp_frame, 0),
                       (void *)pgAliasTemp(ppgi[1].pi_pvp->vp_frame, 1),
                       PAGESIZE);

                ppf->pf_busy = 0;
            }

            if ((err = pgSavePF(pvp, ppf1)) != NO_ERROR)
                goto newpageerror;

            if (reloaded) {
                ppgi[1].pi_pvp->vp_pfidle = 1;
                pgIdlePF(pgFrameToPF( ppgi[1].pi_pvp->vp_frame),
                    ppgi[1].pi_pvp->vp_frame);
            }

            /*
             *  Remove one reference from the old page's data structures
             */
            ppgi[1].pi_pvp->vp_refcount--;
            if (ppgi[1].pi_pte.pt_present) {
                ppgi[1].pi_ppd->pd_pcpresent[ppgi[1].pi_pdi]--;
                if (--(ppgi[1].pi_ppf->pf_refcount) == 0) {
                    ppgi[1].pi_pvp->vp_pfidle = 1;
                    pgIdlePF(ppgi[1].pi_ppf, ppgi[1].pi_pte.pt_frame);
                }
            }

            pgAssert(ppgi[1].pi_pte.pt_dirty == 0);

            /*
             *  Setup the master pte to point to the new vp and whack the
             *  data in the ppgi[1] structure to contain the new info so
             *  we can use the common attach code to take care of the rest.
             */
            ppgi[1].pi_pte.pt_long = (ppgi[1].pi_pte.pt_long &
                                      ~(PT_PRESENT | PT_FRAME | PT_DIRTY)) |
                                     (pgpVPToVPI(pvp) << PT_FRAMESHIFT);
            ppgi[1].pi_ppte->pt_long = ppgi[1].pi_pte.pt_long;
            pgClearSemVP(ppgi[1].pi_pvp);
            ppgi[1].pi_pvp = pvp;
        }
        else {
            /*
             *  If discarable, change to swap-on-write
             */
            if (!swappable) {

                if (ppgi[1].pi_pvp->vp_sow == 0) {
                    if (err = pgCommit(1, PG_SWAPPABLE))
                       goto common_err;
                    pgCommitFree(1, PG_DISCARDABLE);
                    ppgi[1].pi_pvp->vp_sow = 1;
                }
            }

            if ((err = pgSavePF(ppgi[1].pi_pvp, ppf1)) != NO_ERROR)
               goto common_err;

            if (reloaded) {
                ppgi[1].pi_pvp->vp_pfidle = 1;
                pgIdlePF(pgFrameToPF( ppgi[1].pi_pvp->vp_frame),
                    ppgi[1].pi_pvp->vp_frame);
            }

            ppgi[1].pi_pvp->vp_flags |= VP_PRIVATIZED;
        }
    /*
     *  If reloading of privatized pages requested and this page is private
     *  setup for the reload.
     */
    } else if ((ppgi[1].pi_flags & PG_RELOAD) &&
              (ppgi[1].pi_pvp->vp_flags & VP_PRIVATIZED)) {

        /*
         * Set swappable if the page is not discardable, or
         * the page is discardable, and swap-on-write, and has
         * been dirtied (swap-on-load).
         */
        swappable = ((ppgi[1].pi_pvp->vp_discardable == 0) ||
                    ((ppgi[1].pi_pvp->vp_discardable) &&
                    (ppgi[1].pi_pvp->vp_sow == 1) &&
                    (ppgi[1].pi_pvp->vp_flags & VP_DIRTY)));

        if (!swappable) {
            if (err = pgCommit(1, PG_DISCARDABLE))
                return(err);
        }
        else {
            if (err = pgCommit(1, PG_SWAPPABLE))
                return(err);
        }

        pvp = pgAllocVP();
        pvp->vp_flags = VP_BUSY;
        pvp->vp_refcount = 1;
        pvp->vp_obpg = ppgi[1].pi_pvp->vp_obpg;
        pvp->vp_hob = ppgi[1].pi_pvp->vp_hob;
        if (ppgi[1].pi_pvp->vp_pf) {
            pvp->vp_block =
                      pgFrameToPF(ppgi[1].pi_pvp->vp_frame)->pf_block;
        } else {
            pvp->vp_block = ppgi[1].pi_pvp->vp_frame;
        }

        if (!swappable) {
            pvp->vp_sow = 0;
            pvp->vp_flags |= VP_DISCARDABLE;
        }
        else {
            ppf1 = pgFindPF(ppgi[1].pi_pvp, FALSE);

            pgAssert((ppf1 != NULL) && (ppf1->pf_pvp == NULL));

            if (err = pgGetPF(ppgi[0].pi_page, 0, SSToDS(&ppf))) {
                pgFreeVP(pvp);
                pgCommitFree(1, PG_SWAPPABLE);
                return(err);
            }
            ppf->pf_busy = 1;

            ppf->pf_block = pvp->vp_block;
            pvp->vp_frame = BLINK(ppf);

            ppf->pf_pvp = pvp;
            ppf->pf_refcount = 0;

            pvp->vp_pf = 1;

            memcpy((void *)pgAliasTemp(pvp->vp_frame, 0),
                   (void *)pgAliasTemp(pgPFToFrame(ppf1), 1),
                   PAGESIZE);

            ppf->pf_busy = 0;

            pvp->vp_flags |= VP_PFIDLE;
            pgIdlePF(ppf, pvp->vp_frame);
        }

        ppgi[1].pi_pte.pt_long = (ppgi[1].pi_pte.pt_long &
                                  ~(PT_PRESENT | PT_FRAME )) |
                                 (pgpVPToVPI(pvp) << PT_FRAMESHIFT);
        pgClearSemVP(pvp);
        goto common_processing;
    }

    /*
     *  From here on down, the attach is pretty much just copying the
     *  old pte to the new and incrementing appropriate pt counts.
     */
    if (ppgi[1].pi_pte.pt_uvirt == 0) {

        /*
         *  For attachments of not present pages, all we have to do specially
         *  is increment the vp ref count.  present pages require a little
         *  more work.
         */
        ppgi[1].pi_pvp->vp_refcount++;
     retry:
        if (ppgi[1].pi_pte.pt_present) {
            /*
             *  When setting the permissions below, we are going to clear the
             *  dirty (guard) bit.  Propogate it here into the vp so we don't
             *  loose it.  If the pte was illegally written to we might idle
             *  the page, so be sure to jump up to check for that.
             */
             if (ppgi[1].pi_ppte->pt_dirty) {
                 if (ppgi[1].pi_ppte->pt_ring3write == 0) {
                     pgIllegalWrite(ppgi[1].pi_pvp);
                     ppgi[1].pi_pte.pt_long = ppgi[1].pi_ppte->pt_long;  //david
                     goto retry;
                 }
                 ppgi[1].pi_pvp->vp_flags |= (VP_DIRTY | VP_SHDIRTY);
                 if (ppgi[1].pi_pvp->vp_df) {
                     /*
                      *  Don't keep dfs if they have no valid data
                      */
                     ppgi[1].pi_pvp->vp_df = 0;
                     SMFreeDF(ppgi[1].pi_ppf->pf_block);
#ifdef PGSTRICT
                     ppgi[1].pi_ppf->pf_block = -1;
#endif
                 }
             }

            /*
             * guard page requests when the old pte is present is a little more
             * difficult, since we can't just copy the old to the new because
             * guard pages must be not present.  So here we should setup the
             * old pte to look like it is not present so we can attach it
             * by copying the pte below, just like every other page.
             * Note that I am mucking with pi_pte, the image of the pte,
             * not pi_ppte, the real pte.  We do that atomically farther down.
             */
            if (ppgi[1].pi_flags & PG_GUARD) {
                ppgi[1].pi_pte.pt_long &= ~(PT_PRESENT | PT_FRAME);
                ppgi[1].pi_pte.pt_long |= pgpVPToVPI(ppgi[1].pi_pvp) <<
                                          PT_FRAMESHIFT;

            /*
             *  non-guard page attach of a present page.
             */
            } else {
              commonpresent:
                ppgi[0].pi_ppd->pd_pcpresent[ppgi[0].pi_pdi]++;
                ppgi[1].pi_ppf->pf_refcount++;
            }

        /*
         *  For attaches to not present pages with a PF somewhere in core,
         *  mark the new pte present.  Why?  I'll tell you.  1) When starting
         *  a new app, we do a bunch of attaches to dll segments and the
         *  target of the attach is some random context, often something
         *  like harderr which hasn't run in days, so all of its ptes are
         *  marked not present.  This makes all the pages in the new app
         *  not present even if there are some pages in there that all apps
         *  touch.  So this way we make those pages present without the
         *  overhead of faults (fast though they are...); 2) VDMs make aliases
         *  to not present things all the time for EMM and were spending
         *  a lot of time faulting, too.  But skip this optimization if
         *  the caller requested a guard page -- too much work to signal it.
         *  Note that below I am mucking with pi_pte, the image of the pte,
         *  not pi_ppte, the real pte.  We do that atomically farther down.
         */
        } else if ((ppgi[0].pi_flags & PG_GUARD) == 0 &&
                   ppgi[1].pi_pvp->vp_pf) {

            ppgi[1].pi_pte.pt_present = 1;
            ppgi[1].pi_pte.pt_frame = ppgi[1].pi_pvp->vp_frame;
            ppgi[1].pi_ppf = pgFrameToPF(ppgi[1].pi_pvp->vp_frame);
            if (ppgi[1].pi_pvp->vp_pfidle) {
                /*
                 * PF is on the idle list.  Remove it.
                 */
                ppgi[1].pi_pvp->vp_pfidle = 0;
                pgDeletePF(ppgi[1].pi_ppf);  /* clears lock cnt */
                ppgi[1].pi_ppf->pf_refcount = 0;
            }
            goto commonpresent;
        }
    } else if (ppgi[0].pi_flags & PG_GUARD) { /* no uvirt guards */
        return(ERROR_ACCESS_DENIED);
    } else if (ppgi[1].pi_pte.pt_present) {   /* present uvirt */
        ppgi[0].pi_ppd->pd_pcpresent[ppgi[0].pi_pdi]++;
    }

    /*
     *  Now for all of the common processing.  Copy the pte, set the access
     *  permissions, and increment pt allocated count.
    */
common_processing:;
    ppgi[0].pi_ppd->pd_pcalloc[ppgi[0].pi_pdi]++;
    ppgi[0].pi_ppte->pt_long = (ppgi[1].pi_pte.pt_long & ppgi[1].pi_data2) |
                               ppgi[1].pi_data1;
    return(0);
}


#pragma code_seg()
#pragma code_seg("PGSWAP_TEXT", "CODE")

/***EP  PGMove - move page table entries.
 *
 *      PGMove moves page table entries around in the virtual address space.
 *      The non-overlapping portion of the destination must be unallocated
 *      and invalid (0).  The non-overlapping portion of the source range
 *      is zeroed after the move.
 *
 *      All pages in the source range must be allocated and valid.
 *      Locked pages cannot be moved and will cause an internal error.
 *
 *          - pin source and destination page tables, possibly allocing dest
 *          - take source page semaphores
 *          - verify that non-overlapping new region is 0ed
 *          - move ptes and update present and allocated pte counts
 *          - clear source page semaphores
 *          - unpin page tables
 *
 *      ENTRY   pagesrc - source virtual page
 *              pptdasrc - pointer to source ptda
 *              pagedst - destination virtual page
 *              pptdadst - pointer to destination ptda
 *              npages - number of pages
 *              hob - object handle for destination
 *      EXIT    0 on success, else error code
 *r
 *      pin both sets of page tables
 *      take source page semaphores
 *      setup direction of move
 *      verify non-overlapping portion of that destination is unallocated
 *      for (each page in range)
 *          if (page is present)
 *              increment dest pte present count
 *              decrement src pte present count
 *          copy source pte to destination pte
 *          zero source pte
 *      increase dest allocated counts over entire range
 *      decrease src allocated counts over entire range
 *      clear source semaphores
 *      unpin both sets of page tables
 */

int
PGMove(pagesrc, pptdasrc, pagedst, pptdadst, npages, hob)
page_t pagesrc;
struct ptda_s *pptdasrc;
page_t pagedst;
struct ptda_s *pptdadst;
int npages;
ushort_t hob;
{
    register struct pte_s *pptedst;
    register struct pte_s *pptesrc;
    struct pte_s *pptedstbase;
    struct pte_s *pptesrcbase;
    struct pgdata_s *ppddst;
    struct pgdata_s *ppdsrc;
    ulong_t pdisrc;
    ulong_t pdidst;
    int i;
    int delta;
    struct pte_s *ppteverify;
    int err;
    int offset;
    struct vp_s *pvp;
    ushort_t obpg;
    page_t pagesrcbase;
    page_t pagedstbase;

    /*
     *  BUGBUG - bs - this routine is very complicated because it handles
     *  moves of overlapping ranges.  It turns out that no one uses this
     *  feature so this routine could be re-written to be about a 10 line
     *  pgDriver2 worker.  If you do this, make sure you keep this code
     *  around some place, because someone might want to use it some day
     *  (#ifdef it).  The tricky part about making a pgDriver2 worker will
     *  be error clean-up if we die part way through.
     */

    /*
     *  Prelude.  Pin and take source sems (no need to mess with destination
     *  semaphores -- the pages are unallocated, hopefully).   pgTakeSem
     *  will verify that all the source pages are valid.
     */
    pagedstbase = pagedst;
    if (err = pgPinPt(pagedst, pptdadst, npages, PG_ALLOC, SSToDS(&ppddst))){
        return(err);
    }
    pagesrcbase = pagesrc;
    if (err=pgPinPt(pagesrc, pptdasrc, npages, 0, SSToDS(&ppdsrc))){
        goto unpindst;
    }
    pptesrc = pgPagePdTopPte(pagesrc, ppdsrc);
    pptesrcbase = pptesrc;
    if (err = pgTakeSem(pptesrc, npages, 0)) {
        goto unpinboth;
    }
    pptedst = pgPagePdTopPte(pagedst, ppddst);
    pptedstbase = pptedst;

    /*
     *  Setup variables used in main moving loop.  They are:
     *          delta - direction to make move (1 = low to high, -1 = opposite
     *          offset - index of first page to move (low end or high end)
     *          pptesrc, pptedst - set to end of ranges if high to low move
     *          i - number of pages in non-overlapping portion of destination
     *              (number of pages to verify)
     */
    offset = 0;                                 /* assume move low to high */
    delta = 1;
    obpg = 0;
    if (pptdasrc != pptdadst && (pagesrc <MINSYSPAGE || pagedst < MINSYSPAGE)){
        i = npages;                             /* all dst pages need verify*/
    } else if (pagesrc > pagedst) {
        i = min(npages, pagesrc-pagedst);       /* verify non-overlap */
    } else {
        delta = -1;                             /* copy high to low */
        offset = npages - 1;
        pptesrc += offset;
        pptedst += offset;
        pagedst += offset;
        pagesrc += offset;
        i = min(npages, pagedst-pagesrc);       /* verify non-overlap */
        obpg = offset;
    }

    /*
     *  Verify non-overlapping portion of destination is unallocated
     */
    for (ppteverify = pptedst; i--; ppteverify += delta) {
        if (ppteverify->pt_long != 0) {
            err = ERROR_INVALID_ADDRESS;
            pgClearSem(pptesrcbase, npages);
            goto unpinboth;
        }
    }

    /*
     *  The move looop.
     */
    for (i = npages; i--; pptesrc += delta, pptedst += delta, pagesrc += delta,
                          pagedst += delta, obpg += delta) {
        if (pptesrc->pt_long) {

            /*
             *  Update the page table counts
             */
            pdisrc = pgPagePdToPDI(pagesrc, ppdsrc);
            ppdsrc->pd_pcalloc[pdisrc]--;
            pdidst = pgPagePdToPDI(pagedst, ppddst);
            ppddst->pd_pcalloc[pdidst]++;
            if (pptesrc->pt_present) {
                if (pptesrc->pt_resident == 1) {
                    ppddst->pd_pcresident[pdidst]++;
                    ppdsrc->pd_pcresident[pdisrc]--;
                }
                ppddst->pd_pcpresent[pdidst]++;
                ppdsrc->pd_pcpresent[pdisrc]--;
            }


            /*
             *  If the page isn't uvirt, update the hob and obpg in the vp.
             */
            if (pptesrc->pt_uvirt == 0) {
                if (pptesrc->pt_present) {
                    /*
                     *  Can't move locked pages.
                     */
                    pgAssert(LOCKS(pgFrameToPF(pptesrc->pt_frame)) == 0);
                    pvp = pgFrameToPF(pptesrc->pt_frame)->pf_pvp;
                } else {
                    pvp = pgFrameToVP(pptesrc->pt_frame);
                }
                pvp->vp_hob = hob;
                pvp->vp_obpg = obpg;
            }

            /*
             *  Do the actual move by copying the src pte to the dest and
             *  zeroing the source.
             */
            pptedst->pt_long = pptesrc->pt_long;
            pptesrc->pt_long = 0;
        }
    }

    err = 0;
    pgFlushTLB();
    pgClearSem(pptedstbase, npages);
  unpinboth:
    pgUnpinPt(pagesrcbase, ppdsrc, npages, 0);
  unpindst:
    pgUnpinPt(pagedstbase, ppddst, npages, 0);
    return(err);
}

#pragma code_seg()
#pragma code_seg("LOAD_TUNEDIT1_SWP","CODE")


/***EP  PGPreload - moves memory in support of preloads
 *
 *      PGPreload moves physical memory from one virtual address to
 *      another.  This routine is provided for the loader to move newly
 *      read preload pages from a temporary buffer to their final location.
 *      The source and destination virtual memory must be previously
 *      allocated and the source ptes must refer to physical storage
 *      while the destination ptes must be allocated but discarded or
 *      AOD.  After the move, the source ptes are zeroed and overcommit is
 *      reduced as needed.
 *
 *      A flag is is passed to PGPreload to indicate if this is a forced
 *      preload -- fforce.  If set, the pages will be made not present
 *      and swappable as they are moved to reduce memory consumption.
 *      The source ptes in this case must be swappable or in a resident
 *      buffer allocated by PGAllocBuf.
 *
 *      If the call is not in support of a forced preload, the source pages
 *      may be from a PGAllocBuf buffer or discardable.  If discardable,
 *      they are assumed to contain temporary loader block numbers that should
 *      not be moved into the destination ptes if the page has been made
 *      not present between the time they were read and this call.
 *
 *      Moves covering overlapping regions are not supported.
 *
 *      ENTRY   pagesrc - source virtual page, must be in system space
 *              pagedst - destination virtual page
 *              pptdadst - pointer to destination ptda
 *              npages - number of pages
 *              faultpage - page which we already hold the semaphore of
 *                          (because this is a segment load in support of a
 *                          page fault) or -1.
 *              fforce - set if call is for forced preload situation
 *      EXIT    0 on success, else error code
 *
 */
int
PGPreload(pagesrc, pagedst, pptdadst, npages, faultpage, fforce)
page_t pagesrc;
page_t pagedst;
struct ptda_s *pptdadst;
int npages;
page_t faultpage;
int fforce;
{
    ulong_t commitflags = 0;
    int err;
    ulong_t ccommitpages = 0;
    struct pginfo_s pgi[2];
    pgi[1].pi_data2 = 0;

    /*
     *  If source is not in a PGAllocBuf buffer, set the PG_DISCARDABLE
     *  and PG_SWAPPABLE flags so the worker knows to decrement the
     *  page table present and allocated counts.  Also set pi_data2
     *  to 1.
     */
    if (pagesrc < (PGSegBufLaddr >> PAGESHIFT) ||
        pagesrc >= ((PGSegBufLaddr + BUFSIZE) >> PAGESHIFT)) {
        commitflags |= PG_DISCARDABLE | PG_SWAPPABLE;
        pgi[1].pi_data2 = 1;
        pgAssert(faultpage == -1);
    }

    /*
     *  Chose correct commit type (PG_DISCARDABLE or PG_SWAPPABLE).
     */
    if (fforce) {
        commitflags &= ~PG_DISCARDABLE;
    } else {
        commitflags &= ~PG_SWAPPABLE;
    }

    /*
     *  Setup pginfo structures
     */
    pgi[0].pi_page = pagesrc;
    pgi[0].pi_flags = 0;
    pgi[1].pi_page = pagedst;
    pgi[1].pi_pptda = pptdadst;
    pgi[1].pi_flags = 0;
    pgi[1].pi_data1 = (ulong_t) SSToDS(&ccommitpages);
    pgi[1].pi_data3 = fforce;
    if (faultpage != -1) {
        pgi[0].pi_data1 = faultpage;
        pgi[1].pi_flags = PG_SKIPBUSY | PG_SKIPFREE;
    }

    /*
     *  Call pgDriver2 with correct worker.  It will return
     *  in ccommitpages the number of pages that were actually processed
     *  (i.e., that we have to free overcommit for -- unless they were in
     *  an AllocBuf, in which case commit is permanent).
     */
    err = pgDriver2(SSToDS(&pgi), npages, &pgwPreload, 0);

    /*
     *  Free any overcommit from the source buffer if wan't an AllocBuf
     */
    if (commitflags) {
        pgCommitFree(ccommitpages, commitflags&(PG_DISCARDABLE|PG_SWAPPABLE));
    }
    pgFlushTLB();
    return(err);
}


/***LP  pgwPreload - move pages in support of loads
 *
 *      This routine moves a physical page from a system space buffer to
 *      a user address.
 *
 *      This routine is swappable.
 *
 *      ENTRY:  standard pgDriver2 arguments, with the following peculiarities:
 *                  structure 0 is the source, 1 the destination
 *                  [1].pi_data1 = pointer to count of pages operated upon
 *                  [1].pi_data2 = 0 if source is allocbuf, else 1
 *                  [1].pi_data3 = 1 if force-preload
 *
 *      EXIT:   0 if success, else error
 *              *pi_data1++
 *
 *          if (destination is present)
 *              return error
 *          if (source is present)
 *              move pf to destination
 *          free source pte
 */
int
pgwPreload(ppgi)
register struct pginfo_s *ppgi;
{
    struct pf_s *ppfsrc;
    ulong_t framesrc;

    /*
     *  Resident pages only allowed in AllocBufs
     */
    pgAssert(ppgi[0].pi_pte.pt_resident == 0 || ppgi[1].pi_data2 == 0);

    /*
     *  If the caller passed in PG_SKIPBUSY, just skip any pages that already
     *  are loaded.
     */
    if (ppgi[1].pi_pvp->vp_pf) {
        if (ppgi[1].pi_flags & PG_SKIPBUSY) {
            return(0);
        } else {
            return(ERROR_ACCESS_DENIED);
        }
    }

    /*
     *  Attach the source pf to the destination page.  If there isn't a pf,
     *  the page has been discarded and we should ignore it.
     */
    if (ppgi[0].pi_pvp->vp_pf) {
        framesrc = ppgi[0].pi_pvp->vp_frame;
        ppfsrc = pgFrameToPF(framesrc);
        ppfsrc->pf_pvp = ppgi[1].pi_pvp;
        ppfsrc->pf_block = ppgi[1].pi_pvp->vp_frame;
        ppgi[1].pi_pvp->vp_frame = framesrc;
        ppgi[1].pi_pvp->vp_pf = 1;

        if ((ppgi[1].pi_pvp->vp_flags & VP_DISCARDABLE) &&
            ((ppgi[1].pi_pvp->vp_flags & VP_SOW) == 0) )
            pgDiscardableInmem++;

        /*
         *  If not forced preload, make the page present as we move it.
         */
        if (ppgi[1].pi_data3 == 0) {
            if (ppgi[0].pi_pvp->vp_pfidle) {
                ppgi[0].pi_pvp->vp_pfidle = 0;
                pgDeletePF(ppfsrc);
                ppfsrc->pf_refcount = 1;
            }
            ppgi[1].pi_ppte->pt_frame = framesrc;
            ppgi[1].pi_ppte->pt_present = 1;
            ppgi[1].pi_ppd->pd_pcpresent[ppgi[1].pi_pdi]++;

        /*
         *  If force preload, the pte is already setup correctly.  Just
         *  make sure the pf is on the idle list.
         */
        } else {
            if (ppgi[0].pi_pvp->vp_pfidle == 0) {
                ppgi[1].pi_pvp->vp_pfidle = 1;
                pgIdlePF(ppfsrc, framesrc);
            }
        }
    }
    ppgi[1].pi_data1++;

    /*
     *  Now free the source page.  Don't free the vp or decrement pt counts
     *  if it is in an allocbuf
     */
    ppgi[0].pi_ppte->pt_long = 0;
    if (ppgi[1].pi_data2) {
        if ((ppgi[0].pi_pvp->vp_flags & VP_DISCARDABLE) &&
            ((ppgi[0].pi_pvp->vp_flags & VP_SOW) == 0)  )
            pgDiscardableInmem--;
        pgFreeVP(ppgi[0].pi_pvp);
        ppgi[0].pi_ppd->pd_pcalloc[ppgi[0].pi_pdi]--;
        if (ppgi[0].pi_pte.pt_present) {
            ppgi[0].pi_ppd->pd_pcpresent[ppgi[0].pi_pdi]--;
        }
    }
    return(0);
}

#pragma code_seg()


/***LP  pgDriver2 - preparation for page manager routines with 2 ranges
 *
 *      This divides two ranges of pages into chunks that do not cross
 *      a page table boundary, and for each chunk pins both sets of
 *      page tables, takes the page semaphores for the target pages in
 *      the second range, and calls a worker routine.  If PG_REMAP is
 *      specified and a page in the first range is allocated, we free it
 *      before calling the worker.
 *
 *      The worker routine must return 0 if success and an error code
 *      otherwise.  The error code is returned to the caller.
 *      The worker is called with a single argument: an array of
 *      two pginfo structures, one structure describing each range.
 *
 *      The error worker is called only on the first range, using the
 *      parameter sequence of pgDriver().
 *
 *      ENTRY - ppgi - array of two pginfo structures describing the two ranges
 *              npages
 *              pfnworker - pointer to worker function
 *              pfnerror - pointer to error worker function
 *                         0 if no clean-up
 *      EXIT    0 if success, else error code
 *      do
 *          compute per-page table information for each range
 *          pin page table in range 0
 *          pin page table in range 1
 *          for (ever)
 *              if (page in range 1 is allocated)
 *                  take its semaphore
 *                  if (page in range 0 is allocated)
 *                      if (PG_REMAP)
 *                          free page in range 0
 *                      else
 *                          error
 *                          break
 *                  call worker
 *              else if (PG_SKIPFREE == 0)
 *                  error
 *              if (error)
 *                  break
 *              if (at end of range)
 *                  break
 *              increment variables to point to next page
 *              if (either page crosses a page boundary)
 *                  break
 *          unpin page tables for both ranges
 *      while (not at end of range)
 */

int
pgDriver2(ppgi, npages, pfnworker, pfnerror)
register struct pginfo_s *ppgi;
int npages;
pgpfnDriver_t pfnworker;
pgpfnDriver_t pfnerror;
{
    struct pgdata_s *ppd0;
    struct pgdata_s *ppd1;
    int err = 0;
    ulong_t pagebase0 = ppgi[0].pi_page;
    ulong_t i;
    BOOL IgottheVPSem;                               /*  151270 */

    if (pgPageToPd(ppgi[0].pi_page, ppgi[0].pi_pptda, SSToDS(&ppd0)) ||
        pgPageToPd(ppgi[1].pi_page, ppgi[1].pi_pptda, SSToDS(&ppd1))) {
        pgAssert(FALSE);        /* VM should catch these errors */
    }
    ppgi[0].pi_ppd = ppd0;
    ppgi[1].pi_ppd = ppd1;
    ppgi[0].pi_ppte = pgPagePdTopPte(ppgi[0].pi_page, ppd0);
    ppgi[1].pi_ppte = pgPagePdTopPte(ppgi[1].pi_page, ppd1);

    /*
     */
    do {
        /*
         *  Compute page table indexes and pin the page tables
         */
        ppgi[0].pi_pdi = pgPagePdToPDI(ppgi[0].pi_page, ppd0);
        ppgi[1].pi_pdi = pgPagePdToPDI(ppgi[1].pi_page, ppd1);
        pgPinOnePt(ppgi[1].pi_pdi, ppd1, 0, err, 0);
        if (err) {
            /*
             *  If the page table is unallocated and the skipfree flag
             *  was specified, just go on to the next pt.  Otherwise,
             *  return error.
             */
            if (err==ERROR_ACCESS_DENIED && (ppgi[1].pi_flags & PG_SKIPFREE)) {
                i = NPTEPPT - (ppgi[1].pi_page & (NPTEPPT - 1));
                ppgi[0].pi_page += i;
                ppgi[1].pi_page += i;
                npages -= i;
                ppgi[0].pi_ppte += i;
                ppgi[1].pi_ppte += i;
                continue;
            }
            goto undo;
        }
        pgPinOnePt(ppgi[0].pi_pdi,ppd0,(ppgi[0].pi_flags & PG_ALLOC),err,1);
        if (err) {
            goto unpin1;
        }

        /*
         *  Loop over the pages on these two page tables calling the worker
         *  function until we cross a page table boundary, run out of pages,
         *  or get an error
         */
        for (;;) {
            /*
             *  If page1 is already allocated, take its semaphore or
             *  skip it or something
             */
             IgottheVPSem=FALSE;                          /*  151270  */
        retrysem1:
            if (ppgi[1].pi_ppte->pt_long) {
                ppgi[1].pi_pte.pt_long = ppgi[1].pi_ppte->pt_long;
                if (ppgi[1].pi_pte.pt_uvirt == 0) {
                    if (ppgi[1].pi_pte.pt_present) {
                        ppgi[1].pi_ppf =pgFrameToPF(ppgi[1].pi_pte.pt_frame);
                        ppgi[1].pi_pvp = ppgi[1].pi_ppf->pf_pvp;
                    } else {
                        ppgi[1].pi_pvp =pgFrameToVP(ppgi[1].pi_pte.pt_frame);
                    }

                    /*
                     *  If the semaphore is free, take it, else block
                     *  as long as the caller didn't specify PG_SKIPBUSY
                     *  in which case we assume that we already hold the
                     *  the semaphore for the page specified in pgi[0].pi_data1
                     *  and we don't want to wait for any others.
                     *                                                151270
                     *  If the caller already holds the semaphore DO NOT set
                     *  the IgottheVPSem flag TRUE.  In this case it is the
                     *  callers responsiblilty to free the semaphore when it
                     *  removes the page from the idle or free list.  151270
                     */

                    if (ppgi[1].pi_pvp->vp_busy == 0) {
                        pgSetSem(ppgi[1].pi_pvp);
                        IgottheVPSem=TRUE;                     /*  151270  */
                    } else if ((ppgi[1].pi_flags & PG_SKIPBUSY) == 0) {
                        ppgi[1].pi_pvp->vp_wanted = 1;
                        if (err=TKSleep(ppgi[1].pi_pvp, -1, 0, 0)) {
                            break;
                        }
                        goto retrysem1;
                    } else if (ppgi[1].pi_page != ppgi[0].pi_data1) {
                        goto skippage;
                    }
                }

                /*
                 *  At this point, the page is range 1 is allocated and
                 *  we have its semaphore.
                 *
                 *  Now checkout page in range 0.  If it is currently
                 *  allocated free it if PG_REMAP is specified, else error.
                 */
            retrysem0:
                if (ppgi[0].pi_pte.pt_long = ppgi[0].pi_ppte->pt_long) {
                    if (ppgi[0].pi_pte.pt_uvirt == 0) {
                        if (ppgi[0].pi_pte.pt_present) {
                            ppgi[0].pi_ppf=pgFrameToPF(ppgi[0].pi_pte.pt_frame);
                            ppgi[0].pi_pvp = ppgi[0].pi_ppf->pf_pvp;
                        } else {
                            ppgi[0].pi_pvp=pgFrameToVP(ppgi[0].pi_pte.pt_frame);
                        }

                        /*
                         *  If remap was specified, take sem and free the page
                         */
                        if (ppgi[0].pi_flags & PG_REMAP) {
                            /*
                             *  If this page is mapped to the same memory
                             *  as the other range and the PG_SWITCHMAP is
                             *  set: swap pte entries else just skip it
                             */
                            if (ppgi[0].pi_pvp == ppgi[1].pi_pvp) {
                                /*
                                 * at this point we have the vp's semaphore
                                 *   and both pd's are pinned
                                 */
                                if (ppgi[0].pi_flags & PG_SWITCHMAP)
                                {
                                   /*
                                    *  if the both pte's reside in the same pd,
                                    *    don't worry about the pd's count fields
                                    */
                                   if (ppgi[0].pi_pdi != ppgi[1].pi_pdi)
                                   {
                                      /*
                                       *  if the both pte's are present/not present
                                       *    don't worry about the pd's count fields
                                       *    BUGBUG - what about locked pf's
                                       */
                                      if (((ppgi[0].pi_ppte->pt_long) & PT_PRESENT) !=
                                          ((ppgi[1].pi_ppte->pt_long) & PT_PRESENT))
                                      {
                                         if ((ppgi[0].pi_ppte->pt_long) & PT_PRESENT)
                                         {
                                            ppgi[0].pi_ppd-> pd_pcpresent[ppgi[0].pi_pdi]--;
                                            ppgi[1].pi_ppd-> pd_pcpresent[ppgi[1].pi_pdi]++;
                                         } else {
                                            ppgi[0].pi_ppd-> pd_pcpresent[ppgi[0].pi_pdi]++;
                                            ppgi[1].pi_ppd-> pd_pcpresent[ppgi[1].pi_pdi]--;
                                         }
                                      }
                                   }
                                   ppgi[1].pi_ppte->pt_long = ppgi[0].pi_pte.pt_long;
                                   ppgi[0].pi_ppte->pt_long = ppgi[1].pi_pte.pt_long;
                                }
                                goto errorclearsem;
                            }

                            /*
                             *  If the semaphore is busy, block until it isn't
                             */
                            if (ppgi[0].pi_pvp->vp_busy == 1) {
                                if (err=TKSleep(ppgi[0].pi_pvp, -1, 0, 0)) {
                                    break;
                                } else {
                                   goto retrysem0;
                                }
                            }
                            pgAssert(ppgi[0].pi_flags & PG_ALLOC); /* no commit */
                            goto freeit;
                        }
                    } else if (ppgi[0].pi_flags & PG_REMAP) {
                      freeit:
                        pgwFree(&ppgi[0]);
                        ppgi[0].pi_pte.pt_long = 0;
                    }

                /*
                 *  If the page is unallocated, PG_ALLOC must have been spec'ed
                 */
                } else if ((ppgi[0].pi_flags & PG_ALLOC) == 0) {
                    err = ERROR_ACCESS_DENIED;
                    goto errorclearsem;
                }

                /*
                 *  call worker and clear semaphore for the 2nd range
                 */
                err=(*pfnworker)(ppgi);

              errorclearsem:
              /*
               *  If and only if the VP semaphore was obtained in this routine
               *  can we safetly free the semaphore here.  Otherwise  it was
               *  obtained by our caller and will remain in place until the
               *  caller clears it.                                 151270
               */

                if (ppgi[1].pi_pte.pt_uvirt == 0  &&            /*  151270 */
                    (IgottheVPSem))                             /*  151270 */
                    pgClearSemVP(ppgi[1].pi_pvp);
                if (err) {
                    break;
                }

            /*
             *  If the page in range 1 is unallocated, PG_SKIPFREE
             *  better be specified
             */
            } else if ((ppgi[1].pi_flags & PG_SKIPFREE) == 0) {
                err = ERROR_ACCESS_DENIED;
                break;
            }

            /*
             *  We are now done with the page we were messing with.
             *  Advance to the next one.
             *  If we are at the end of the range, break out.
             */
          skippage:
            if (--npages == 0) {
                break;
            }
            ++ppgi[0].pi_ppte;
            ++ppgi[1].pi_ppte;
            ++ppgi[0].pi_page;
            ++ppgi[1].pi_page;

            /*
             *  If we cross a page table boundary, break out of the loop so
             *  we can pin the next page table.
             */
            if ((ppgi[1].pi_page & (NPTEPPT - 1)) == 0 ||
                (ppgi[0].pi_page & (NPTEPPT - 1)) == 0) {
                break;
            }
        }

        /*
         *  Here we have broken out of our for (each page) loop because of
         *  error, we crossed a page table boundary, or we are at the end.
         *  Do the right thing.
         */
        pgUnpinOnePt(ppgi[0].pi_pdi, ppd0, 0, 1);
      unpin1:
        pgUnpinOnePt(ppgi[1].pi_pdi, ppd1, 0, 1);
        if (err) {
            goto undo;
        }
    } while (npages > 0);

    /*
     *  If we fall out of here, we went through all the pages without error.
     */
    return(0);

  undo:
    /*
     *  If we get an error somewhere along the line, undo whatever
     *  work we have done so far, assuming we have done some and
     *  an error worker was specified.
     *  Any operations on the current page table are assumed to be already
     *  cleaned-up.
     *  Undo takes place only on range 1.
     */
    npages = ppgi[0].pi_page - pagebase0;
    ppgi[0].pi_page = pagebase0;
    ppgi[0].pi_flags |= PG_SKIPFREE | PG_IGNOREINTS;
    if (npages && pfnerror && pgDriver(ppgi, npages, pfnerror, 0)) {
        pgAssert(FALSE);
    }
    return(err);
}
