src/Pure/Isar/outer_lex.ML
author wenzelm
Sat, 21 Jan 2006 23:02:14 +0100
changeset 18728 6790126ab5f6
parent 18547 d1978038b945
child 19305 5c16895d548b
permissions -rw-r--r--
simplified type attribute;
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
     1
(*  Title:      Pure/Isar/outer_lex.ML
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
     2
    ID:         $Id$
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
     3
    Author:     Markus Wenzel, TU Muenchen
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
     4
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
     5
Outer lexical syntax for Isabelle/Isar.
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
     6
*)
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
     7
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
     8
signature OUTER_LEX =
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
     9
sig
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    10
  datatype token_kind =
7477
c7caea1ce78c removed text vars;
wenzelm
parents: 7026
diff changeset
    11
    Command | Keyword | Ident | LongIdent | SymIdent | Var | TypeIdent | TypeVar |
17164
a786e1a1ce02 added AltString token (delimited by ASCII back-quotes);
wenzelm
parents: 17069
diff changeset
    12
    Nat | String | AltString | Verbatim | Space | Comment | Sync | Malformed | EOF
15143
05b5995f214e Make token an eqtype to assist reconstructing input
aspinall
parents: 14991
diff changeset
    13
  eqtype token
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    14
  val str_of_kind: token_kind -> string
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    15
  val stopper: token * (token -> bool)
6859
2b3db2b6c129 sync token;
wenzelm
parents: 6743
diff changeset
    16
  val not_sync: token -> bool
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    17
  val not_eof: token -> bool
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    18
  val position_of: token -> Position.T
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    19
  val pos_of: token -> string
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    20
  val is_kind: token_kind -> token -> bool
7026
69724548fad1 separate command tokens;
wenzelm
parents: 6859
diff changeset
    21
  val keyword_with: (string -> bool) -> token -> bool
16029
070ed43b86f8 added ident_with;
wenzelm
parents: 15531
diff changeset
    22
  val ident_with: (string -> bool) -> token -> bool
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    23
  val name_of: token -> string
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    24
  val is_proper: token -> bool
9130
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
    25
  val is_semicolon: token -> bool
17069
ee08b2466a09 clarify is_newline vs. is_blank;
wenzelm
parents: 16029
diff changeset
    26
  val is_comment: token -> bool
8580
e79ee31d3936 added is_begin/end_ignore;
wenzelm
parents: 8231
diff changeset
    27
  val is_begin_ignore: token -> bool
e79ee31d3936 added is_begin/end_ignore;
wenzelm
parents: 8231
diff changeset
    28
  val is_end_ignore: token -> bool
17069
ee08b2466a09 clarify is_newline vs. is_blank;
wenzelm
parents: 16029
diff changeset
    29
  val is_blank: token -> bool
8651
f095f3b8181a added is_newline;
wenzelm
parents: 8580
diff changeset
    30
  val is_newline: token -> bool
14991
26fb63c4acb5 added unparse;
wenzelm
parents: 14981
diff changeset
    31
  val unparse: token -> string
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    32
  val val_of: token -> string
5876
273056b673ec replaced is_symid by is_sid;
wenzelm
parents: 5825
diff changeset
    33
  val is_sid: string -> bool
9130
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
    34
  val !!! : string -> (Position.T * 'a -> 'b) -> Position.T * 'a -> 'b
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
    35
  val incr_line: ('a -> 'b * 'c) -> Position.T * 'a -> 'b * (Position.T * 'c)
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
    36
  val keep_line: ('a -> 'b * 'c) -> Position.T * 'a -> 'b * (Position.T * 'c)
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
    37
  val scan_blank: Position.T * Symbol.symbol list
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
    38
    -> Symbol.symbol * (Position.T * Symbol.symbol list)
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
    39
  val scan_string: Position.T * Symbol.symbol list -> string * (Position.T * Symbol.symbol list)
7026
69724548fad1 separate command tokens;
wenzelm
parents: 6859
diff changeset
    40
  val scan: (Scan.lexicon * Scan.lexicon) ->
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    41
    Position.T * Symbol.symbol list -> token * (Position.T * Symbol.symbol list)
7026
69724548fad1 separate command tokens;
wenzelm
parents: 6859
diff changeset
    42
  val source: bool -> (unit -> (Scan.lexicon * Scan.lexicon)) ->
69724548fad1 separate command tokens;
wenzelm
parents: 6859
diff changeset
    43
    Position.T -> (Symbol.symbol, 'a) Source.source ->
7682
46de8064c93c added Space, Comment token kinds (keep actual text);
wenzelm
parents: 7477
diff changeset
    44
    (token, Position.T * (Symbol.symbol, 'a) Source.source) Source.source
17164
a786e1a1ce02 added AltString token (delimited by ASCII back-quotes);
wenzelm
parents: 17069
diff changeset
    45
  val source_proper: (token, 'a) Source.source ->
a786e1a1ce02 added AltString token (delimited by ASCII back-quotes);
wenzelm
parents: 17069
diff changeset
    46
    (token, (token, 'a) Source.source) Source.source
9130
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
    47
  val make_lexicon: string list -> Scan.lexicon
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    48
end;
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    49
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    50
structure OuterLex: OUTER_LEX =
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    51
struct
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    52
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    53
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    54
(** tokens **)
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    55
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    56
(* datatype token *)
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    57
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    58
datatype token_kind =
7477
c7caea1ce78c removed text vars;
wenzelm
parents: 7026
diff changeset
    59
  Command | Keyword | Ident | LongIdent | SymIdent | Var | TypeIdent | TypeVar |
17164
a786e1a1ce02 added AltString token (delimited by ASCII back-quotes);
wenzelm
parents: 17069
diff changeset
    60
  Nat | String | AltString | Verbatim | Space | Comment | Sync | Malformed | EOF;
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    61
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    62
datatype token = Token of Position.T * (token_kind * string);
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    63
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    64
val str_of_kind =
7026
69724548fad1 separate command tokens;
wenzelm
parents: 6859
diff changeset
    65
 fn Command => "command"
69724548fad1 separate command tokens;
wenzelm
parents: 6859
diff changeset
    66
  | Keyword => "keyword"
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    67
  | Ident => "identifier"
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    68
  | LongIdent => "long identifier"
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    69
  | SymIdent => "symbolic identifier"
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    70
  | Var => "schematic variable"
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    71
  | TypeIdent => "type variable"
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    72
  | TypeVar => "schematic type variable"
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    73
  | Nat => "number"
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    74
  | String => "string"
17164
a786e1a1ce02 added AltString token (delimited by ASCII back-quotes);
wenzelm
parents: 17069
diff changeset
    75
  | AltString => "back-quoted string"
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    76
  | Verbatim => "verbatim text"
7682
46de8064c93c added Space, Comment token kinds (keep actual text);
wenzelm
parents: 7477
diff changeset
    77
  | Space => "white space"
46de8064c93c added Space, Comment token kinds (keep actual text);
wenzelm
parents: 7477
diff changeset
    78
  | Comment => "comment text"
6859
2b3db2b6c129 sync token;
wenzelm
parents: 6743
diff changeset
    79
  | Sync => "sync marker"
10748
74ed77fa5310 recover: malformed result;
wenzelm
parents: 9195
diff changeset
    80
  | Malformed => "bad input"
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    81
  | EOF => "end-of-file";
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    82
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    83
10748
74ed77fa5310 recover: malformed result;
wenzelm
parents: 9195
diff changeset
    84
(* control tokens *)
6859
2b3db2b6c129 sync token;
wenzelm
parents: 6743
diff changeset
    85
2b3db2b6c129 sync token;
wenzelm
parents: 6743
diff changeset
    86
fun not_sync (Token (_, (Sync, _))) = false
2b3db2b6c129 sync token;
wenzelm
parents: 6743
diff changeset
    87
  | not_sync _ = true;
2b3db2b6c129 sync token;
wenzelm
parents: 6743
diff changeset
    88
10748
74ed77fa5310 recover: malformed result;
wenzelm
parents: 9195
diff changeset
    89
val malformed = Token (Position.none, (Malformed, ""));
15224
1bd35fd87963 Allow scanning to recover and reconstruct bad input
aspinall
parents: 15209
diff changeset
    90
fun malformed_of xs = Token (Position.none, (Malformed, implode xs));
10748
74ed77fa5310 recover: malformed result;
wenzelm
parents: 9195
diff changeset
    91
6859
2b3db2b6c129 sync token;
wenzelm
parents: 6743
diff changeset
    92
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    93
(* eof token *)
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    94
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    95
val eof = Token (Position.none, (EOF, ""));
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    96
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    97
fun is_eof (Token (_, (EOF, _))) = true
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    98
  | is_eof _ = false;
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
    99
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   100
val stopper = (eof, is_eof);
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   101
val not_eof = not o is_eof;
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   102
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   103
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   104
(* get position *)
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   105
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   106
fun position_of (Token (pos, _)) = pos;
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   107
val pos_of = Position.str_of o position_of;
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   108
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   109
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   110
(* kind of token *)
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   111
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   112
fun is_kind k (Token (_, (k', _))) = k = k';
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   113
7026
69724548fad1 separate command tokens;
wenzelm
parents: 6859
diff changeset
   114
fun keyword_with pred (Token (_, (Keyword, x))) = pred x
69724548fad1 separate command tokens;
wenzelm
parents: 6859
diff changeset
   115
  | keyword_with _ _ = false;
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   116
16029
070ed43b86f8 added ident_with;
wenzelm
parents: 15531
diff changeset
   117
fun ident_with pred (Token (_, (Ident, x))) = pred x
070ed43b86f8 added ident_with;
wenzelm
parents: 15531
diff changeset
   118
  | ident_with _ _ = false;
070ed43b86f8 added ident_with;
wenzelm
parents: 15531
diff changeset
   119
7682
46de8064c93c added Space, Comment token kinds (keep actual text);
wenzelm
parents: 7477
diff changeset
   120
fun is_proper (Token (_, (Space, _))) = false
46de8064c93c added Space, Comment token kinds (keep actual text);
wenzelm
parents: 7477
diff changeset
   121
  | is_proper (Token (_, (Comment, _))) = false
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   122
  | is_proper _ = true;
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   123
9195
29f1e53f9937 fixed is_semicolon (keyword instead of command!);
wenzelm
parents: 9155
diff changeset
   124
fun is_semicolon (Token (_, (Keyword, ";"))) = true
9130
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   125
  | is_semicolon _ = false;
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   126
17069
ee08b2466a09 clarify is_newline vs. is_blank;
wenzelm
parents: 16029
diff changeset
   127
fun is_comment (Token (_, (Comment, _))) = true
ee08b2466a09 clarify is_newline vs. is_blank;
wenzelm
parents: 16029
diff changeset
   128
  | is_comment _ = false;
ee08b2466a09 clarify is_newline vs. is_blank;
wenzelm
parents: 16029
diff changeset
   129
8580
e79ee31d3936 added is_begin/end_ignore;
wenzelm
parents: 8231
diff changeset
   130
fun is_begin_ignore (Token (_, (Comment, "<"))) = true
e79ee31d3936 added is_begin/end_ignore;
wenzelm
parents: 8231
diff changeset
   131
  | is_begin_ignore _ = false;
e79ee31d3936 added is_begin/end_ignore;
wenzelm
parents: 8231
diff changeset
   132
e79ee31d3936 added is_begin/end_ignore;
wenzelm
parents: 8231
diff changeset
   133
fun is_end_ignore (Token (_, (Comment, ">"))) = true
e79ee31d3936 added is_begin/end_ignore;
wenzelm
parents: 8231
diff changeset
   134
  | is_end_ignore _ = false;
e79ee31d3936 added is_begin/end_ignore;
wenzelm
parents: 8231
diff changeset
   135
8651
f095f3b8181a added is_newline;
wenzelm
parents: 8580
diff changeset
   136
17069
ee08b2466a09 clarify is_newline vs. is_blank;
wenzelm
parents: 16029
diff changeset
   137
(* blanks and newlines -- space tokens obey lines *)
8651
f095f3b8181a added is_newline;
wenzelm
parents: 8580
diff changeset
   138
17069
ee08b2466a09 clarify is_newline vs. is_blank;
wenzelm
parents: 16029
diff changeset
   139
fun is_blank (Token (_, (Space, s))) = not (String.isSuffix "\n" s)
ee08b2466a09 clarify is_newline vs. is_blank;
wenzelm
parents: 16029
diff changeset
   140
  | is_blank _ = false;
ee08b2466a09 clarify is_newline vs. is_blank;
wenzelm
parents: 16029
diff changeset
   141
ee08b2466a09 clarify is_newline vs. is_blank;
wenzelm
parents: 16029
diff changeset
   142
fun is_newline (Token (_, (Space, s))) = String.isSuffix "\n" s
8651
f095f3b8181a added is_newline;
wenzelm
parents: 8580
diff changeset
   143
  | is_newline _ = false;
f095f3b8181a added is_newline;
wenzelm
parents: 8580
diff changeset
   144
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   145
14991
26fb63c4acb5 added unparse;
wenzelm
parents: 14981
diff changeset
   146
(* token content *)
9155
adfa40218e06 OuterLex.name_of: include val;
wenzelm
parents: 9130
diff changeset
   147
adfa40218e06 OuterLex.name_of: include val;
wenzelm
parents: 9130
diff changeset
   148
fun name_of (tok as Token (_, (k, x))) =
10748
74ed77fa5310 recover: malformed result;
wenzelm
parents: 9195
diff changeset
   149
  if is_semicolon tok then "terminator"
74ed77fa5310 recover: malformed result;
wenzelm
parents: 9195
diff changeset
   150
  else if x = "" then str_of_kind k
74ed77fa5310 recover: malformed result;
wenzelm
parents: 9195
diff changeset
   151
  else str_of_kind k ^ " " ^ quote x;
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   152
18547
d1978038b945 unparse String/AltString: escape quotes;
wenzelm
parents: 17164
diff changeset
   153
fun escape q =
d1978038b945 unparse String/AltString: escape quotes;
wenzelm
parents: 17164
diff changeset
   154
  implode o map (fn s => if s = q orelse s = "\\" then "\\" ^ s else s) o Symbol.explode;
d1978038b945 unparse String/AltString: escape quotes;
wenzelm
parents: 17164
diff changeset
   155
14991
26fb63c4acb5 added unparse;
wenzelm
parents: 14981
diff changeset
   156
fun unparse (Token (_, (kind, x))) =
26fb63c4acb5 added unparse;
wenzelm
parents: 14981
diff changeset
   157
  (case kind of
18547
d1978038b945 unparse String/AltString: escape quotes;
wenzelm
parents: 17164
diff changeset
   158
    String => x |> quote o escape "\""
d1978038b945 unparse String/AltString: escape quotes;
wenzelm
parents: 17164
diff changeset
   159
  | AltString => x |> enclose "`" "`" o escape "`"
14991
26fb63c4acb5 added unparse;
wenzelm
parents: 14981
diff changeset
   160
  | Verbatim => x |> enclose "{*" "*}"
26fb63c4acb5 added unparse;
wenzelm
parents: 14981
diff changeset
   161
  | Comment => x |> enclose "(*" "*)"
26fb63c4acb5 added unparse;
wenzelm
parents: 14981
diff changeset
   162
  | _ => x);
26fb63c4acb5 added unparse;
wenzelm
parents: 14981
diff changeset
   163
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   164
fun val_of (Token (_, (_, x))) = x;
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   165
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   166
fun token_leq (Token (_, (_, x)), Token (_, (_, x'))) = x <= x';
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   167
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   168
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   169
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   170
(** scanners **)
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   171
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   172
fun change_prompt scan = Scan.prompt "# " scan;
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   173
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   174
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   175
(* diagnostics *)
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   176
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   177
fun lex_err msg ((pos, cs), _) = "Outer lexical error" ^ Position.str_of pos ^ ": " ^ msg cs;
9130
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   178
fun !!! msg scan = Scan.!! (lex_err (K msg)) scan;
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   179
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   180
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   181
(* line numbering *)
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   182
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   183
fun incr_line scan = Scan.depend (fn pos => scan >> pair (Position.inc pos));
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   184
val keep_line = Scan.lift;
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   185
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   186
val scan_blank =
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   187
  incr_line ($$ "\n") ||
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   188
  keep_line (Scan.one Symbol.is_blank);
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   189
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   190
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   191
(* scan symbolic idents *)
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   192
17164
a786e1a1ce02 added AltString token (delimited by ASCII back-quotes);
wenzelm
parents: 17069
diff changeset
   193
val sym_chars = explode "!#$%&*+-/:<=>?@^_|~";
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   194
fun is_sym_char s = s mem sym_chars;
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   195
8231
fa93309ff27e symid: include single symbolic char;
wenzelm
parents: 7902
diff changeset
   196
val scan_symid =
fa93309ff27e symid: include single symbolic char;
wenzelm
parents: 7902
diff changeset
   197
  Scan.any1 is_sym_char >> implode ||
fa93309ff27e symid: include single symbolic char;
wenzelm
parents: 7902
diff changeset
   198
  Scan.one Symbol.is_symbolic;
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   199
8231
fa93309ff27e symid: include single symbolic char;
wenzelm
parents: 7902
diff changeset
   200
fun is_symid str =
fa93309ff27e symid: include single symbolic char;
wenzelm
parents: 7902
diff changeset
   201
  (case try Symbol.explode str of
15531
08c8dad8e399 Deleted Library.option type.
skalberg
parents: 15224
diff changeset
   202
    SOME [s] => Symbol.is_symbolic s orelse is_sym_char s
08c8dad8e399 Deleted Library.option type.
skalberg
parents: 15224
diff changeset
   203
  | SOME ss => forall is_sym_char ss
8231
fa93309ff27e symid: include single symbolic char;
wenzelm
parents: 7902
diff changeset
   204
  | _ => false);
fa93309ff27e symid: include single symbolic char;
wenzelm
parents: 7902
diff changeset
   205
5876
273056b673ec replaced is_symid by is_sid;
wenzelm
parents: 5825
diff changeset
   206
val is_sid = is_symid orf Syntax.is_identifier;
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   207
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   208
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   209
(* scan strings *)
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   210
17164
a786e1a1ce02 added AltString token (delimited by ASCII back-quotes);
wenzelm
parents: 17069
diff changeset
   211
local
a786e1a1ce02 added AltString token (delimited by ASCII back-quotes);
wenzelm
parents: 17069
diff changeset
   212
a786e1a1ce02 added AltString token (delimited by ASCII back-quotes);
wenzelm
parents: 17069
diff changeset
   213
fun scan_str q =
7682
46de8064c93c added Space, Comment token kinds (keep actual text);
wenzelm
parents: 7477
diff changeset
   214
  scan_blank ||
9130
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   215
  keep_line ($$ "\\") |-- !!! "bad escape character in string"
17164
a786e1a1ce02 added AltString token (delimited by ASCII back-quotes);
wenzelm
parents: 17069
diff changeset
   216
      (scan_blank || keep_line ($$ q || $$ "\\")) ||
a786e1a1ce02 added AltString token (delimited by ASCII back-quotes);
wenzelm
parents: 17069
diff changeset
   217
  keep_line (Scan.one (not_equal q andf not_equal "\\" andf
6859
2b3db2b6c129 sync token;
wenzelm
parents: 6743
diff changeset
   218
    Symbol.not_sync andf Symbol.not_eof));
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   219
17164
a786e1a1ce02 added AltString token (delimited by ASCII back-quotes);
wenzelm
parents: 17069
diff changeset
   220
fun scan_strs q =
a786e1a1ce02 added AltString token (delimited by ASCII back-quotes);
wenzelm
parents: 17069
diff changeset
   221
  keep_line ($$ q) |--
9130
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   222
    !!! "missing quote at end of string"
17164
a786e1a1ce02 added AltString token (delimited by ASCII back-quotes);
wenzelm
parents: 17069
diff changeset
   223
      (change_prompt ((Scan.repeat (scan_str q) >> implode) --| keep_line ($$ q)));
a786e1a1ce02 added AltString token (delimited by ASCII back-quotes);
wenzelm
parents: 17069
diff changeset
   224
a786e1a1ce02 added AltString token (delimited by ASCII back-quotes);
wenzelm
parents: 17069
diff changeset
   225
in
a786e1a1ce02 added AltString token (delimited by ASCII back-quotes);
wenzelm
parents: 17069
diff changeset
   226
a786e1a1ce02 added AltString token (delimited by ASCII back-quotes);
wenzelm
parents: 17069
diff changeset
   227
val scan_string = scan_strs "\"";
a786e1a1ce02 added AltString token (delimited by ASCII back-quotes);
wenzelm
parents: 17069
diff changeset
   228
val scan_alt_string = scan_strs "`";
a786e1a1ce02 added AltString token (delimited by ASCII back-quotes);
wenzelm
parents: 17069
diff changeset
   229
a786e1a1ce02 added AltString token (delimited by ASCII back-quotes);
wenzelm
parents: 17069
diff changeset
   230
end;
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   231
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   232
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   233
(* scan verbatim text *)
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   234
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   235
val scan_verb =
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   236
  scan_blank ||
6743
5d50225637c8 changed {| |} verbatim syntax to {* *} in order to simplify ProofGeneral setup;
wenzelm
parents: 5876
diff changeset
   237
  keep_line ($$ "*" --| Scan.ahead (Scan.one (not_equal "}"))) ||
6859
2b3db2b6c129 sync token;
wenzelm
parents: 6743
diff changeset
   238
  keep_line (Scan.one (not_equal "*" andf Symbol.not_sync andf Symbol.not_eof));
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   239
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   240
val scan_verbatim =
6743
5d50225637c8 changed {| |} verbatim syntax to {* *} in order to simplify ProofGeneral setup;
wenzelm
parents: 5876
diff changeset
   241
  keep_line ($$ "{" -- $$ "*") |--
9130
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   242
    !!! "missing end of verbatim text"
6743
5d50225637c8 changed {| |} verbatim syntax to {* *} in order to simplify ProofGeneral setup;
wenzelm
parents: 5876
diff changeset
   243
      (change_prompt ((Scan.repeat scan_verb >> implode) --| keep_line ($$ "*" -- $$ "}")));
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   244
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   245
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   246
(* scan space *)
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   247
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   248
val is_space = Symbol.is_blank andf not_equal "\n";
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   249
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   250
val scan_space =
7682
46de8064c93c added Space, Comment token kinds (keep actual text);
wenzelm
parents: 7477
diff changeset
   251
  (keep_line (Scan.any1 is_space) -- Scan.optional (incr_line ($$ "\n")) "" ||
46de8064c93c added Space, Comment token kinds (keep actual text);
wenzelm
parents: 7477
diff changeset
   252
    keep_line (Scan.any is_space) -- incr_line ($$ "\n")) >> (fn (cs, c) => implode cs ^ c);
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   253
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   254
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   255
(* scan nested comments *)
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   256
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   257
val scan_cmt =
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   258
  Scan.lift scan_blank ||
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   259
  Scan.depend (fn d => keep_line ($$ "(" ^^ $$ "*") >> pair (d + 1)) ||
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   260
  Scan.depend (fn 0 => Scan.fail | d => keep_line ($$ "*" ^^ $$ ")") >> pair (d - 1)) ||
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   261
  Scan.lift (keep_line ($$ "*" --| Scan.ahead (Scan.one (not_equal ")")))) ||
6859
2b3db2b6c129 sync token;
wenzelm
parents: 6743
diff changeset
   262
  Scan.lift (keep_line (Scan.one (not_equal "*" andf Symbol.not_sync andf Symbol.not_eof)));
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   263
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   264
val scan_comment =
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   265
  keep_line ($$ "(" -- $$ "*") |--
9130
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   266
    !!! "missing end of comment"
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   267
      (change_prompt
7682
46de8064c93c added Space, Comment token kinds (keep actual text);
wenzelm
parents: 7477
diff changeset
   268
        (Scan.pass 0 (Scan.repeat scan_cmt >> implode) --| keep_line ($$ "*" -- $$ ")")));
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   269
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   270
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   271
(* scan token *)
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   272
9130
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   273
fun scan (lex1, lex2) =
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   274
  let
9130
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   275
    val scanner = Scan.state :-- (fn pos =>
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   276
      let
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   277
        fun token k x = Token (pos, (k, x));
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   278
        fun sync _ = token Sync Symbol.sync;
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   279
      in
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   280
        scan_string >> token String ||
17164
a786e1a1ce02 added AltString token (delimited by ASCII back-quotes);
wenzelm
parents: 17069
diff changeset
   281
        scan_alt_string >> token AltString ||
9130
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   282
        scan_verbatim >> token Verbatim ||
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   283
        scan_space >> token Space ||
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   284
        scan_comment >> token Comment ||
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   285
        keep_line (Scan.one Symbol.is_sync >> sync) ||
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   286
        keep_line (Scan.max token_leq
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   287
          (Scan.max token_leq
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   288
            (Scan.literal lex1 >> (token Keyword o implode))
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   289
            (Scan.literal lex2 >> (token Command o implode)))
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   290
          (Syntax.scan_longid >> token LongIdent ||
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   291
            Syntax.scan_id >> token Ident ||
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   292
            Syntax.scan_var >> token Var ||
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   293
            Syntax.scan_tid >> token TypeIdent ||
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   294
            Syntax.scan_tvar >> token TypeVar ||
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   295
            Syntax.scan_nat >> token Nat ||
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   296
            scan_symid >> token SymIdent))
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   297
      end) >> #2;
14729
0e987111a17e changed Symbol.beginning;
wenzelm
parents: 10748
diff changeset
   298
  in !! (lex_err (fn cs => "bad input " ^ quote (Symbol.beginning 10 cs))) scanner end;
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   299
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   300
9130
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   301
(* token sources *)
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   302
6859
2b3db2b6c129 sync token;
wenzelm
parents: 6743
diff changeset
   303
val is_junk = (not o Symbol.is_blank) andf Symbol.not_sync andf Symbol.not_eof;
15224
1bd35fd87963 Allow scanning to recover and reconstruct bad input
aspinall
parents: 15209
diff changeset
   304
fun recover xs = (keep_line (Scan.any is_junk) >> (fn ts => [malformed_of ts])) xs;
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   305
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   306
fun source do_recover get_lex pos src =
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   307
  Source.source' pos Symbol.stopper (Scan.bulk (fn xs => scan (get_lex ()) xs))
15531
08c8dad8e399 Deleted Library.option type.
skalberg
parents: 15224
diff changeset
   308
    (if do_recover then SOME recover else NONE) src;
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   309
9130
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   310
fun source_proper src = src |> Source.filter is_proper;
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   311
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   312
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   313
(* lexicons *)
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   314
ff8789b49d2e added !!!;
wenzelm
parents: 9051
diff changeset
   315
val make_lexicon = Scan.make_lexicon o map Symbol.explode;
5825
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   316
24e4b1780d33 Outer lexical syntax for Isabelle/Isar.
wenzelm
parents:
diff changeset
   317
end;