-- (C) Copyright International Business Machines Corporation 23 January 
-- 1990.  All Rights Reserved. 
--  
-- See the file USERAGREEMENT distributed with this software for full 
-- terms and conditions of use. 
-- File: tokinternal.d
-- Author: Andy Lowry
-- SCCS Info: @(#)tokinternal.d	1.2 3/13/90

-- Definitions used internally by the tokenizer

tokInternal: using ()
definitions
  
  -- A set of characters forming a character class
  charClass: table of char {init} keys (*);
  
  -- A mapping from chars to chars.. used for associating open and
  -- close brackets
  charMap: table of charPair {full} keys (first);
  charPair: record (
    first: char,
    second: char
  );
  
  -- states in the tokenizer state machine
  tokenState: enumeration ('start', 'inWord', 'inQuote', 'sawQuote');
  
  -- information for a bracketed string currently being scanned
  -- (stacked while scanning embedded bracketed strings)
  tokenizeStack: ordered table of tokenizeStackEntry {full};
  tokenizeStackEntry: record (
    brackets: charPair		-- brackets for this bracketed string token
  );

end definitions
