;       SCCSID = @(#)dbcs.asm	6.1 90/11/17
SCCSID	equ	"@(#)dbcs.asm	13.15 90/09/17"

	PAGE	,132
TITLE	DBCS	Double Byte Character Set support code

;***	Double Byte Character Set support code
;
;	KCD, Microsoft, 9/86

;***	DBCS support package is a set of routines to manage double
;	byte character sets.  Routines are provide to detect
;	double byte characters and manage information
;	about the current set of DBCS lead bytes.
;
;	General Background:
;
;	Computers in the United States use ASCII to represent character.
;	This works because there are about 100 interesting characters
;	in the written alphabet.  Foreign languages (particularly the
;	far east) do not share this trait.  Chinese has over
;	100,000 different characters.  One byte cannot represent all
;	these characters.  These characters are represented by two bytes.
;	The first byte (called the lead byte) is alway greater than
;	127d.  This way it is possible to differentiate between
;	an ASCII character and a lead byte to double byte character.
;	The second byte (the trailing byte) must be greater than
;	39h and less than FDh except 7Fh.  7Fh cannot be a trailing
;	byte.  This scheme permits about 30,000 characters to be
;	represented.  Not all characters but enough to get by.
;
;	Text written in these languages is a mixture of single and
;	double byte characters.  This presents some problems to DOS.
;	An example helps explain.  Suppose we are give string:
;
;	----------------------------------------------------------
;	|    'a'    |	'b'   |	   98h	  |   5C    |	  '\0'	 |
;	----------------------------------------------------------
;	|--single---|-single--|---double byte ------|---single---|
;
;	The trouble comes when dos scans looking for path seperator
;	characters.  The trailing byte of the the double byte
;	character is ascii '\'.  Scanning the character for '\'
;	will incorrectly identify the fourth byte as a path
;	character.
;
;	MORE later.
;
;	The names significant to DOS like LPT1 and CON don't change
;	across languages.
;
;	Data Structures:
;
;	*	One bit of the CharType array is used to determine
;		if a character is a lead byte.
;	*	The CP_Flgs variable of the PTDA is used to determine
;		if a given process has any DBCS activity.
;
;		Country_Code	Global variable of current country code
;		Lead byte tables
;
;	Routines:
;
;	    TestDBCSLOW	  - tells if byte is a valid lead byte.
;	    FindChar	  - Scan a string looking for a character while
;			    watching for DBCS characters
;	    FindMultiChar - scan for a set of characters
;	    ExmDBCS	  - test if byte in string is 2nd byte of
;			    DBCS character
;	    InitKernelCountryData - initialize the country data.
;	    InitKernelDBCS - initialize the kernel DBCS data structures.
;	    DBCSTableInit  -
;	    InitCharTypeHIGH2 -
;
;
;	Modification history:
;
;	KCD	9/12/86
;		File created
;
;	KCD	9/27/86
;		Added FindChar routine
;
;	KCD	10/9/86
;		Move TestDBCS, ExmDBCS, and FindChar to low code
;
;	KCD	10/12/86
;		Added country and lead byte variables.	Added w_IQCtryCode,
;		InitCountryData,  and DBCSTableInit.  Use CharType array
;		to determine if character is LeadByte.
;
;	BWS	11/12/86
;		Optimize TestDBCS and ExmDBCS.
;
;	KCD	1/25/86
;		Modified file to support codepages.
;
;	KCD	2/27/87
;		Removed w_IQCtryCode.


.xlist
include dosseg.inc
include basemaca.inc
include osmaca.inc
include fnofs.inc
include mi.inc
include cdib.inc
include dbcs.inc
include dossymc.inc
.list

	ModName , FNO_DBCS_ASM

;
;	External definitions
;

	EXTRNFAR	MemMapAlias
	EXTRNFAR	MemUnmapAlias
	EXTRNFAR	SegGetInfo

	ReferTask	CP_CaseMapTbl,DWORD
	ReferTask	CP_Flgs,BYTE

	ReferGlobal	DBCS_Codepg,WORD
	ReferGlobal	CDIB_selector,WORD
	ReferGlobal	CharType,byte

	ReferGDT	GDT_PTDA,WORD	; Ring 0 stack alias

;
;	Constants and country codes
;

SIZELBT		EQU	10		; maximum size of the lead byte table

USACTRYCODE	EQU	1		; U.S. is the default

CONSTANTS	SEGMENT

;
;	Country_Code is the country code as set by the user in the
;	config.sys file.  The value is set at system initialization
;	time and does NOT change.  The default country is USA
;

PUBLIC	Country_Code
Country_Code DW USACTRYCODE

;
;	LeadByteTable is the table of valid first bytes of double
;	byte characters.  The list consist of ordered byte pairs
;	that are the inclusive boundries of the lead bytes.  Two
;	bytes of 00h terminate the list.  The table is initialized
;	during system initialization.  Once set the values
;	do not change.	The table is used returned by int21 function
;	63h.  NullLeadByteTable is a label for a non-DBCS codepage
;	lead byte table.  It is just two bytes of zeros.  It over
;	laps the end of the LeadByteTable.

PUBLIC	LeadByteTable
PUBLIC	NullLeadByteTable
LeadByteTable	db (SIZELBT-2) dup(0)
NullLeadByteTable db 2 dup(0)

CONSTANTS	ENDS


	FARCODE LOW
CODE	SEGMENT

    assume ds:nothing,es:nothing,ss:nothing


BREAK <TestDBCSLOW - Test if byte is a valid lead byte>
;***	TestDBCSLOW - Test if byte is a valid lead byte
;
;	TestDBCS is give a byte from a character string.  TestDBCS
;	determines if the byte is a valid lead byte.  Lead bytes
;	are identified by the fDBCS bit in the CharType array (if
;	a lead byte then the fDBCS bit is clear).
;
;	ATTENTION:  Use the TestDBCS or TestDBCSLeadByte macro (see dbcs.inc)
;		    to check if a byte is a DBCS lead byte.  These macros
;		    generate in-line code when called from High2Code,
;		    else they generate calls to this routine.
;		    By the time we get to this routine, we know that the
;		    current process is DBCS-active, so a check on CP_Flgs
;		    for DBCS activity is not performed.
;
;		    Every mode in the kernel must use one of these macros
;		    to check if a byte is the lead byte of a DBCS pair.
;
;	ENTRY	(AL) = byte from character string
;
;	EXIT	Zero set   (ZR) - lead byte of DBCS character
;		Zero clear (NZ) - not lead byte of a DBCS character
;
;	USES	AH
;

Procedure TestDBCSLOW,<hybrid, far16>
    assume ds:nothing, es:nothing, ss:nothing

	MOV	AH,0			; (AX) = index into CharType
	XCHG	AX,BX			; (BX) = index into CharType
	SaveReg <DS>
	DosContext <DS>,,NOASSUME
	TEST	BYTE PTR DS:[BX].CharType,fDBCS
	RestoreReg <DS>
	XCHG	BX,AX			; Restore (AX),(BX) contents
	ret

EndProc TestDBCSLOW


BREAK	<FindChar - Scan a string looking for a character>
;***	FindChar - Scan a string looking for a character.
;
;	In non DBCS code the following sequence is used to find
;	a character in a string:
;		mov	al, '.'
;		les	di, StringPtr
;		mov	cx, length
;		repne	scasb
;
;	With double byte character this sequence will not work.
;	It is possible to have a DBCS with the character as the trailing
;	byte value. This routine is does the DBCS aware equivalent
;	of repe scasb.
;		mov	al, '.'
;		les	di, StringPtr
;		mov	cx, length
;		call	FindChar
;
;	ENTRY	ES:DI points to beginning of text string
;		CX	length of string
;		AL	Search character.
;
;	EXIT	Zero set: character found.  ES:DI points to character
;			CX has number of characters remaining in string.
;		Zero Clear: character not found in string.
;			CX is zero.
;
;	USES	CX, DI, Flags
;
;	NOTES	The procedure does not match the repnz scasb instruction
;		exactly.  If CX is zero on entry the zero flag is reset.
;		The repnz scasb instruction does not affect the flags
;		in this case.
;

Procedure FindChar,<near,faronly,far16>
	ASSUME	DS:NOTHING,ES:NOTHING,SS:TASKAREA

	SaveReg <AX,BX>
	xchg	bl, al		; (BL) = byte we are looking for
fc0:	jcxz	fc2
	mov	al, es:[di]	; get character
	inc	di
	dec	cx
	TestDBCS		; double byte character?
	jz	fc1		; yes -- jump down
	cmp	bl, al		; character we want?
	jz	fc3		; yes -- return
	jmp	short fc0

;	deal with double byte character

fc1:	jcxz	fc2
	inc	di		; skip trailing byte
	dec	cx
	jmp	short fc0

fc2:	cmp	cx, 1		; clear the zero flag
fc3:	RestoreReg <BX,AX>
	ret

EndProc FindChar

CODE	ENDS


	FARCODE HIGH2
HIGH2CODE SEGMENT


;***	FindMultiChar - find first occurence of a char
;
;	Finds the first occurrence of any one of a set of characters in an
;	ASCIIZ text taking into account DBCS in the source string. NULL
;	and DBCS characters cannot be searched for. This routine assumes the
;	input pointers are valid.
;
;	This is called by FSH_FINDCHAR.
;
;	ENTRY	bx     = number of characters to search for
;		ds:si -> ASCIIZ text to search in
;		es:dx -> characters to search for
;
;	EXIT	if 'NC' - success
;		  ds:si -> first character found
;		if 'C' - character not found
;
;	USES	ax, bx, cx, dx, si, di, flags
;
;	PSEUDOCODE
;
;	if (bx != 0) {
;	  while (ds:[si] != '\0') {
;	    if (ds:[si] != DBCS lead byte)
;	      if (ds:[si] appears in es:[dx] string) {
;		return carry clear
;	      } else
;		si++
;	    else
;	      si+=2
;	  }
;	}
;	return carry set

Procedure FindMultiChar,<hybrid>
   assume ds:nothing, es:nothing, ss:TASKAREA
	or	bx,bx			; anything to search for?
	jz	fmc40			; no, error exit

; ds:si -> text to examine
; es:dx -> characters to search for
; bx = number of characters to search for

	TestDBCSProcess			; Does this process use DBCS ?
	jz	fmc50			; Yes, check for each char being DBCS
					; lead byte

; No DBCS; do it the fast way

fmc10:	mov	al,ds:[si]		; al = char or DBCS lead byte

	or	al,al			; end of string?
	jz	fmc40			; yes, nothing found

; al = not end of string, see if it's a wanted character

	mov	di,dx			; es:di -> characters to search for
	mov	cx,bx			; cx = # chars to search for
	repne	scasb			; it is there?
	je	fmc30			; yes, return result to caller

	inc	si			; advance to next char in text
	jmp	short fmc10

; ds:si -> character found, carry already cleared

fmc30:	ret

fmc40:	stc				; return error
	ret

; DBCS process; while processing, check for each char being DBCS

fmc50:	mov	al,ds:[si]		; al = char or DBCS lead byte
	TestDBCSLeadByte		; lead byte?
	jz	fmc60			; yes, skip it

	or	al,al			; end of string?
	jz	fmc40			; yes, nothing found

; al = not DBCS and not end of string, see if it's a wanted character

	mov	di,dx			; es:di -> characters to search for
	mov	cx,bx			; cx = # chars to search for
	repne	scasb			; it is there?
	je	fmc30			; yes, return result to caller

	inc	si			; advance to next char in text
	jmp	short fmc50

fmc60:	add	si,2			; skip DBCS char
	jmp	short fmc50

EndProc FindMultiChar


Break	<ExmDBCS - Check if byte is 2nd byte of DBCS>
;***	ExmDBCS - Check if byte is 2nd byte of DBCS.
;
;	ENTRY	ES:BX points to beginning of text string
;		ES:DX points to a BYTE to be examined in the string
;
;	EXIT	Carry Set: byte is 2nd byte of DBCS.
;		Carry Clear: bytes is NOT 2nd byte of DBCS
;
;	USES	Flags
;
;	CALLS	InternalError
;

Procedure ExmDBCS,near
	ASSUME	DS:NOTHING,ES:NOTHING

	test	[CP_Flgs], fCP_DBCS	; any DBCS activity for this process?
	jz	exm0			; yes - jump down to do all the work
	ret				; no - return with carry clear

exm0:	SaveReg <BX,DI>
	CMP	BX,DX
	JAE	exm5			; No scanning necessary

	MOV	DI,BX			; (ES:DI) = start of string
	MOV	BH,0

exm1:	MOV	BL,ES:[DI]		; (BL) = character to test
	TEST	BYTE PTR CS:CharTypeHIGH2[BX],fDBCS
	JNZ	exm2			; Not DBCS lead byte

	INC	DI			; Skip over lead byte

exm2:	INC	DI			; Skip to next character
	CMP	DI,DX
	JB	exm1			; More byte(s) to scan
	JA	exm4			; Past char, so was 2nd byte

	CLC				; At char, so not 2nd byte
exm3:	RestoreReg  <DI,BX>
	ret

exm4:	STC
	JMP	exm3

exm5:	JE	exm3			; At start of string, no 2nd byte

	InternalError <ExmDBCS>		; BX > DX.  Should NEVER happen.

EndProc ExmDBCS


;*  CharTypeHIGH2 - Copy of CharType table from DOSGROUP
;
;   Used to optimize TestDBCS.

	PUBLIC	CharTypeHIGH2
CharTypeHIGH2	DB	256 DUP (?)

HIGH2CODE ENDS


	 FARCODE INIT
INITCODE segment

;
;	DBCS Initialization
;
;	After the initialization code builds the CDIB it is
;	necessary to initialize the kernel data structures
;	to properly handle DBCS characters.  This is done
;	by InitKernelDBCS.  InitKernelCountryData is
;	called to initialize the country code data structures.
;


;***	InitKernelCountryData	- Initialize country data structures.
;
;	Initialize the DOS data structures to reflect the country code
;	found in the CDIB.  The following data structures are
;	initialized:
;		Country_Code
;		3xBox country code data structures
;
;	ENTRY	NONE
;
;	EXIT	NONE
;
;	USES	Flags
;
;	CALLS	NONE
;
Procedure InitKernelCountryData,NEAR
assume ds:nothing,es:nothing,ss:TASKAREA

	pusha
	SaveReg <DS,ES>
	DosContext	DS
	mov	ax, [CDIB_selector]		; done cdib exist?
	or	ax, ax
	jz	ikcd2				; no - jump to return
	mov	es, ax
	mov	di, es:[CDIB_country_ptr]
	mov	bx, es:[di].CDIB_ct_code	; get country number
	mov	[Country_Code], bx		; save in DOSGROUP

ikcd2:	RestoreReg <ES,DS>
	popa
	ret

EndProc InitKernelCountryData


;***	InitKernelDBCS	- Initialize kernel DBCS data structures.
;
;	Initialize the DOS data structures to reflect the DBCS codepages
;	in the system.	The following data structures are initialized:
;
;		DBCS_Codepg	The number of the DBCS codepage in the
;				system. 0FFFFh if not DBCS codepage.
;		LeadByteTable	Lead byte table of DBCS codepage if
;				any.
;		CharType array	fDBCS bit is cleared for lead bytes of
;				DBCS codepage.
;		CP_Flgs		fCP_DBCS bit is cleared if primary codepage
;				is a DBCS codepage.
;		CP_CaseMapTbl	in the PTDA is set to point to the proper
;				casemap table for the primary codepage.
;
;	ENTRY	None
;
;	EXIT	Nothing is returned
;
;	USES	Flags
;
;	CALLS	DBCSTableInit
;
;	NOTES	CP/DOS 1.1 only allows one double byte codepage in the
;		system at one time.
;
Procedure InitKernelDBCS,NEAR
assume ds:nothing,es:nothing,ss:TASKAREA

	pusha
	SaveReg <DS,ES>

	DosContext	DS
	mov	ax, [CDIB_selector]	; set CDIB segment
	or	ax, ax			; defined?
	jz	ikd5			; no, just return
	mov	ds, ax
	ASSUME	DS:NOTHING
	mov	si, ds:[CDIB_codepage_ptr]	; get pointer to codepages
	mov	cx, ds:[si].CDIB_cp_number_codepages	; get number of codepgs
	or	cx,cx					; if zero change to 1
	jnz	ikd1
	inc	cx
ikd1:	add	si, CDIB_cp_first_id		; DS:SI => first codepage
	SaveReg <ds,si>				; save pointer to first codepg

	; scan through codepages looking for a DBCS codepage

ikd2:	mov	di, [si].CDIB_cp_data_ptr	; DS:DI => codepg data
	mov	di, ds:[di].CDIB_cpd_DBCS_ptr	; DS:DI => lead byte tbl
	add	di,CDIB_DBCS_first_range
	mov	ax, ds:[di]			; get first word of table
	or	ax, ax				; any lead bytes
	jnz	ikd6				; yes - dbcs codepage
	add	si, SIZE CDIB_cp_id_section	; no - advance to next codepage
	loop	ikd2
ikd4:	RestoreReg <SI, DS>			; get pointer to first codepg

;	Set up PTDA variable to point to various codepage structures.
;	*** NOTE: this is a duplicate of code found in w_SetCP.  If
;	this code is updated the code in w_SetCP must also be updated. ***

;	set pointer to casemap table

	mov	di, ds:[si].CDIB_cp_data_ptr
	mov	di, ds:[di].CDIB_cpd_casemap_ptr
	sub	di, 7Eh				;save sub on every character
	mov	ss:[CP_CaseMapTbl].Offst,di
	mov	ss:[CP_CaseMapTbl].Segmt,ds	;set pointer to casemap table

;	set the PTDA flag indicating dbcs activity if necessary.

	DosContext ES
	or	[CP_Flgs], fCP_DBCS		;assume no DBCS, set flag
	mov	bx, ds:[si].CDIB_cp_id		;get codepage number
	cmp	[DBCS_Codepg], bx		;codepage DBCS active?
	jnz	ikd5				;no -- done
	and	[CP_Flgs],NOT fCP_DBCS		;yes -- clear the flag
ikd5:	RestoreReg <ES,DS>
	popa
	ret

;	found a dbcs codepage -- initialize necessary data structures

ikd6:	DosContext	ES
	mov	ax, ds:[si].CDIB_cp_id		; get codepage number
	mov	[DBCS_Codepg], ax		; save it in DosGroup

;	Copy lead byte table into DosGroup for 3xbox

	mov	si,di				; DS:SI => lead byte table in CP
	mov	di, OFFSET DOSGROUP:LeadByteTable
ikd8:	lodsw					; need data in AX, don't use
	stosw					;    a movsw instruction
	or	ax, ax
	jnz	ikd8
	call	DBCSTableInit
	jmp	short ikd4

EndProc InitKernelDBCS


;***	DBCSTableInit - Initialize the dbcs table
;
;	TestDBCS uses a field in the CharType array to determine
;	if a byte is a lead byte.  The fDBCS field of the CharType
;	array in initialized with the information in the lead
;	byte table.
;
;	ENTRY	NONE
;
;	EXIT	fDBCS field of the CharType array is initialized.
;
;	USES	AX, CX, SI, DI, DS, ES, and flags
;
;	CALLS	NONE
;
;	NOTE:	Procedure assume all fDBCS bits of CharType array
;		are set before the call.  The procedure only clears
;		the appropriate bits.  This means this procedure
;		can only be called once.  In the future if we need
;		to call the procedure more than once set all the
;		fDBCS bits at the beginning of the procedure.
;

Procedure   DBCSTableInit,NEAR
	ASSUME	DS:NOTHING,ES:NOTHING

	DosContext <ds>,ax
	mov	si,offset LeadByteTable	; (ds:si) -> lead byte table

DTI_2:	lodsw
	or	ax, ax			; two byte zero -- we're done
	jz	DTI_6			; done
	xor	cx, cx
	mov	cl, ah			; cx = ending byte
	xor	ah, ah			; ax = starting byte
	sub	cx, ax			; cx = number of bytes
	inc	cx
	mov	di, ax			; es:[di].CharType = byte of interest
DTI_4:	and	[CharType+di],not fDBCS	; Clear DBCS bit
	inc	di
	loop	DTI_4
	jmp	short	DTI_2		; get next pair of bytes

;*  Copy DosGroup:CharType table to High2Code:CharTypeHIGH2

DTI_6:	call	InitCharTypeHIGH2
	ret

EndProc DBCSTableInit


;***	InitCharTypeHIGH2 - Copy CharType table to High2Code
;
;	ENTRY:	CLD (direction flag clear)
;
;	EXIT:	High2Code:CharTypeHIGH2 filled in from DosGroup:CharType
;
;	USES:	All but BP

    ASSUME  DS:NOTHING,ES:NOTHING

Procedure   InitCharTypeHIGH2,near

	MOV	AX,DOSHIGH2CODE		; (AX) = DOSHIGH2CODE selector
	XOR	DX,DX			; (DX) = current PTDA
	CALLFAR SegGetInfo
	CALLFAR MemMapAlias		; (AX) = writeable selector to High2Code
	jc	icthie
	MOV	ES,AX			    ; (ES:DI) = destination
	MOV	DI,Offset DOSHIGH2CODE:CharTypeHIGH2
	DosContext  <DS>
	MOV	SI,Offset DOSGROUP:CharType ; (DS:SI) = source
	MOV	CX,256/2		; (CX) = number of words in table
	REP MOVSW			; Copy table
	CALLFAR MemUnmapAlias		; Unmap alias
	ret
icthie: Panic <'InitCharTypeHIGH2:MemMapAlias'>
EndProc	    InitCharTypeHIGH2


INITCODE ends
	END
