src/Pure/General/symbol.scala
author wenzelm
Fri, 30 Aug 2013 23:38:18 +0200
changeset 53337 b3817a0e3211
parent 53316 c3e549e0d3c7
child 53400 673eb869e6ee
permissions -rw-r--r--
sort items according to persistent history of frequency of use;
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
27901
28083e9f8d1d Basic support for Isabelle symbols.
wenzelm
parents:
diff changeset
     1
/*  Title:      Pure/General/symbol.scala
28083e9f8d1d Basic support for Isabelle symbols.
wenzelm
parents:
diff changeset
     2
    Author:     Makarius
28083e9f8d1d Basic support for Isabelle symbols.
wenzelm
parents:
diff changeset
     3
27924
8dd8b564faf5 tuned comments;
wenzelm
parents: 27923
diff changeset
     4
Detecting and recoding Isabelle symbols.
27901
28083e9f8d1d Basic support for Isabelle symbols.
wenzelm
parents:
diff changeset
     5
*/
28083e9f8d1d Basic support for Isabelle symbols.
wenzelm
parents:
diff changeset
     6
28083e9f8d1d Basic support for Isabelle symbols.
wenzelm
parents:
diff changeset
     7
package isabelle
28083e9f8d1d Basic support for Isabelle symbols.
wenzelm
parents:
diff changeset
     8
36011
3ff725ac13a4 adapted to Scala 2.8.0 Beta1 -- with notable changes to scala.collection;
wenzelm
parents: 34316
diff changeset
     9
import scala.collection.mutable
31522
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
    10
import scala.util.matching.Regex
48922
6f3ccfa7818d more precise counting of line/column;
wenzelm
parents: 48775
diff changeset
    11
import scala.annotation.tailrec
27901
28083e9f8d1d Basic support for Isabelle symbols.
wenzelm
parents:
diff changeset
    12
28083e9f8d1d Basic support for Isabelle symbols.
wenzelm
parents:
diff changeset
    13
31522
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
    14
object Symbol
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
    15
{
43696
58bb7ca5c7a2 explicit indication of type Symbol.Symbol;
wenzelm
parents: 43695
diff changeset
    16
  type Symbol = String
58bb7ca5c7a2 explicit indication of type Symbol.Symbol;
wenzelm
parents: 43695
diff changeset
    17
58bb7ca5c7a2 explicit indication of type Symbol.Symbol;
wenzelm
parents: 43695
diff changeset
    18
43418
c69e9fbb81a8 recovered markup for non-alphabetic keywords;
wenzelm
parents: 40529
diff changeset
    19
  /* ASCII characters */
c69e9fbb81a8 recovered markup for non-alphabetic keywords;
wenzelm
parents: 40529
diff changeset
    20
c69e9fbb81a8 recovered markup for non-alphabetic keywords;
wenzelm
parents: 40529
diff changeset
    21
  def is_ascii_letter(c: Char): Boolean = 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z'
c69e9fbb81a8 recovered markup for non-alphabetic keywords;
wenzelm
parents: 40529
diff changeset
    22
  def is_ascii_digit(c: Char): Boolean = '0' <= c && c <= '9'
c69e9fbb81a8 recovered markup for non-alphabetic keywords;
wenzelm
parents: 40529
diff changeset
    23
  def is_ascii_quasi(c: Char): Boolean = c == '_' || c == '\''
c69e9fbb81a8 recovered markup for non-alphabetic keywords;
wenzelm
parents: 40529
diff changeset
    24
c69e9fbb81a8 recovered markup for non-alphabetic keywords;
wenzelm
parents: 40529
diff changeset
    25
  def is_ascii_letdig(c: Char): Boolean =
c69e9fbb81a8 recovered markup for non-alphabetic keywords;
wenzelm
parents: 40529
diff changeset
    26
    is_ascii_letter(c) || is_ascii_digit(c) || is_ascii_quasi(c)
c69e9fbb81a8 recovered markup for non-alphabetic keywords;
wenzelm
parents: 40529
diff changeset
    27
c69e9fbb81a8 recovered markup for non-alphabetic keywords;
wenzelm
parents: 40529
diff changeset
    28
  def is_ascii_identifier(s: String): Boolean =
50238
98d35a7368bd more uniform Symbol.is_ascii_identifier in ML/Scala;
wenzelm
parents: 50233
diff changeset
    29
    s.length > 0 && is_ascii_letter(s(0)) && s.forall(is_ascii_letdig)
43418
c69e9fbb81a8 recovered markup for non-alphabetic keywords;
wenzelm
parents: 40529
diff changeset
    30
c69e9fbb81a8 recovered markup for non-alphabetic keywords;
wenzelm
parents: 40529
diff changeset
    31
48775
92ceb058391f simplified symbol matching;
wenzelm
parents: 48774
diff changeset
    32
  /* symbol matching */
27901
28083e9f8d1d Basic support for Isabelle symbols.
wenzelm
parents:
diff changeset
    33
48775
92ceb058391f simplified symbol matching;
wenzelm
parents: 48774
diff changeset
    34
  private val symbol_total = new Regex("""(?xs)
92ceb058391f simplified symbol matching;
wenzelm
parents: 48774
diff changeset
    35
    [\ud800-\udbff][\udc00-\udfff] | \r\n |
92ceb058391f simplified symbol matching;
wenzelm
parents: 48774
diff changeset
    36
    \\ < (?: \^raw: [\x20-\x7e\u0100-\uffff && [^.>]]* | \^? ([A-Za-z][A-Za-z0-9_']*)? ) >? |
92ceb058391f simplified symbol matching;
wenzelm
parents: 48774
diff changeset
    37
    .""")
27924
8dd8b564faf5 tuned comments;
wenzelm
parents: 27923
diff changeset
    38
48775
92ceb058391f simplified symbol matching;
wenzelm
parents: 48774
diff changeset
    39
  private def is_plain(c: Char): Boolean =
92ceb058391f simplified symbol matching;
wenzelm
parents: 48774
diff changeset
    40
    !(c == '\r' || c == '\\' || Character.isHighSurrogate(c))
48773
0e1bab274672 more liberal scanning of potentially malformed symbols;
wenzelm
parents: 48704
diff changeset
    41
0e1bab274672 more liberal scanning of potentially malformed symbols;
wenzelm
parents: 48704
diff changeset
    42
  def is_malformed(s: Symbol): Boolean =
0e1bab274672 more liberal scanning of potentially malformed symbols;
wenzelm
parents: 48704
diff changeset
    43
    s.length match {
0e1bab274672 more liberal scanning of potentially malformed symbols;
wenzelm
parents: 48704
diff changeset
    44
      case 1 =>
0e1bab274672 more liberal scanning of potentially malformed symbols;
wenzelm
parents: 48704
diff changeset
    45
        val c = s(0)
0e1bab274672 more liberal scanning of potentially malformed symbols;
wenzelm
parents: 48704
diff changeset
    46
        Character.isHighSurrogate(c) || Character.isLowSurrogate(c) || c == '\ufffd'
0e1bab274672 more liberal scanning of potentially malformed symbols;
wenzelm
parents: 48704
diff changeset
    47
      case 2 =>
0e1bab274672 more liberal scanning of potentially malformed symbols;
wenzelm
parents: 48704
diff changeset
    48
        val c1 = s(0)
0e1bab274672 more liberal scanning of potentially malformed symbols;
wenzelm
parents: 48704
diff changeset
    49
        val c2 = s(1)
0e1bab274672 more liberal scanning of potentially malformed symbols;
wenzelm
parents: 48704
diff changeset
    50
        !(c1 == '\r' && c2 == '\n' || Character.isSurrogatePair(c1, c2))
48774
c4bd5bb3ae69 further clarification of malformed symbols;
wenzelm
parents: 48773
diff changeset
    51
      case _ => !s.endsWith(">") || s == "\\<>" || s == "\\<^>"
48773
0e1bab274672 more liberal scanning of potentially malformed symbols;
wenzelm
parents: 48704
diff changeset
    52
    }
34137
6cc9a0cbaf55 refined some Symbol operations/signatures;
wenzelm
parents: 34134
diff changeset
    53
43696
58bb7ca5c7a2 explicit indication of type Symbol.Symbol;
wenzelm
parents: 43695
diff changeset
    54
  def is_physical_newline(s: Symbol): Boolean =
43675
8252d51d70e2 simplified Symbol.iterator: produce strings, which are mostly preallocated;
wenzelm
parents: 43511
diff changeset
    55
    s == "\n" || s == "\r" || s == "\r\n"
38877
682c4932b3cc Command.newlines: account for physical newlines;
wenzelm
parents: 38479
diff changeset
    56
34137
6cc9a0cbaf55 refined some Symbol operations/signatures;
wenzelm
parents: 34134
diff changeset
    57
  class Matcher(text: CharSequence)
6cc9a0cbaf55 refined some Symbol operations/signatures;
wenzelm
parents: 34134
diff changeset
    58
  {
48775
92ceb058391f simplified symbol matching;
wenzelm
parents: 48774
diff changeset
    59
    private val matcher = symbol_total.pattern.matcher(text)
34137
6cc9a0cbaf55 refined some Symbol operations/signatures;
wenzelm
parents: 34134
diff changeset
    60
    def apply(start: Int, end: Int): Int =
6cc9a0cbaf55 refined some Symbol operations/signatures;
wenzelm
parents: 34134
diff changeset
    61
    {
6cc9a0cbaf55 refined some Symbol operations/signatures;
wenzelm
parents: 34134
diff changeset
    62
      require(0 <= start && start < end && end <= text.length)
34316
f879b649ac4c clarified Symbol.is_plain/is_wellformed -- is_closed was rejecting plain backslashes;
wenzelm
parents: 34193
diff changeset
    63
      if (is_plain(text.charAt(start))) 1
34138
4008c2f5a46e refined some Symbol operations/signatures;
wenzelm
parents: 34137
diff changeset
    64
      else {
34137
6cc9a0cbaf55 refined some Symbol operations/signatures;
wenzelm
parents: 34134
diff changeset
    65
        matcher.region(start, end).lookingAt
6cc9a0cbaf55 refined some Symbol operations/signatures;
wenzelm
parents: 34134
diff changeset
    66
        matcher.group.length
6cc9a0cbaf55 refined some Symbol operations/signatures;
wenzelm
parents: 34134
diff changeset
    67
      }
6cc9a0cbaf55 refined some Symbol operations/signatures;
wenzelm
parents: 34134
diff changeset
    68
    }
31522
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
    69
  }
27937
fdf77e7be01a more robust pattern: look at longer matches first, added catch-all case;
wenzelm
parents: 27935
diff changeset
    70
fdf77e7be01a more robust pattern: look at longer matches first, added catch-all case;
wenzelm
parents: 27935
diff changeset
    71
43695
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
    72
  /* iterator */
33998
fc56cfc6906e added elements: Interator;
wenzelm
parents: 31929
diff changeset
    73
43696
58bb7ca5c7a2 explicit indication of type Symbol.Symbol;
wenzelm
parents: 43695
diff changeset
    74
  private val char_symbols: Array[Symbol] =
43675
8252d51d70e2 simplified Symbol.iterator: produce strings, which are mostly preallocated;
wenzelm
parents: 43511
diff changeset
    75
    (0 until 256).iterator.map(i => new String(Array(i.toChar))).toArray
8252d51d70e2 simplified Symbol.iterator: produce strings, which are mostly preallocated;
wenzelm
parents: 43511
diff changeset
    76
43696
58bb7ca5c7a2 explicit indication of type Symbol.Symbol;
wenzelm
parents: 43695
diff changeset
    77
  def iterator(text: CharSequence): Iterator[Symbol] =
58bb7ca5c7a2 explicit indication of type Symbol.Symbol;
wenzelm
parents: 43695
diff changeset
    78
    new Iterator[Symbol]
40522
wenzelm
parents: 40443
diff changeset
    79
    {
43489
132f99cc0a43 tuned iteration over short symbols;
wenzelm
parents: 43488
diff changeset
    80
      private val matcher = new Matcher(text)
132f99cc0a43 tuned iteration over short symbols;
wenzelm
parents: 43488
diff changeset
    81
      private var i = 0
132f99cc0a43 tuned iteration over short symbols;
wenzelm
parents: 43488
diff changeset
    82
      def hasNext = i < text.length
132f99cc0a43 tuned iteration over short symbols;
wenzelm
parents: 43488
diff changeset
    83
      def next =
132f99cc0a43 tuned iteration over short symbols;
wenzelm
parents: 43488
diff changeset
    84
      {
132f99cc0a43 tuned iteration over short symbols;
wenzelm
parents: 43488
diff changeset
    85
        val n = matcher(i, text.length)
43675
8252d51d70e2 simplified Symbol.iterator: produce strings, which are mostly preallocated;
wenzelm
parents: 43511
diff changeset
    86
        val s =
8252d51d70e2 simplified Symbol.iterator: produce strings, which are mostly preallocated;
wenzelm
parents: 43511
diff changeset
    87
          if (n == 0) ""
8252d51d70e2 simplified Symbol.iterator: produce strings, which are mostly preallocated;
wenzelm
parents: 43511
diff changeset
    88
          else if (n == 1) {
8252d51d70e2 simplified Symbol.iterator: produce strings, which are mostly preallocated;
wenzelm
parents: 43511
diff changeset
    89
            val c = text.charAt(i)
8252d51d70e2 simplified Symbol.iterator: produce strings, which are mostly preallocated;
wenzelm
parents: 43511
diff changeset
    90
            if (c < char_symbols.length) char_symbols(c)
8252d51d70e2 simplified Symbol.iterator: produce strings, which are mostly preallocated;
wenzelm
parents: 43511
diff changeset
    91
            else text.subSequence(i, i + n).toString
8252d51d70e2 simplified Symbol.iterator: produce strings, which are mostly preallocated;
wenzelm
parents: 43511
diff changeset
    92
          }
8252d51d70e2 simplified Symbol.iterator: produce strings, which are mostly preallocated;
wenzelm
parents: 43511
diff changeset
    93
          else text.subSequence(i, i + n).toString
43489
132f99cc0a43 tuned iteration over short symbols;
wenzelm
parents: 43488
diff changeset
    94
        i += n
132f99cc0a43 tuned iteration over short symbols;
wenzelm
parents: 43488
diff changeset
    95
        s
132f99cc0a43 tuned iteration over short symbols;
wenzelm
parents: 43488
diff changeset
    96
      }
33998
fc56cfc6906e added elements: Interator;
wenzelm
parents: 31929
diff changeset
    97
    }
43489
132f99cc0a43 tuned iteration over short symbols;
wenzelm
parents: 43488
diff changeset
    98
44949
b49d7f1066c8 Symbol.explode as in ML;
wenzelm
parents: 44238
diff changeset
    99
  def explode(text: CharSequence): List[Symbol] = iterator(text).toList
b49d7f1066c8 Symbol.explode as in ML;
wenzelm
parents: 44238
diff changeset
   100
48922
6f3ccfa7818d more precise counting of line/column;
wenzelm
parents: 48775
diff changeset
   101
  def advance_line_column(pos: (Int, Int), text: CharSequence): (Int, Int) =
6f3ccfa7818d more precise counting of line/column;
wenzelm
parents: 48775
diff changeset
   102
  {
6f3ccfa7818d more precise counting of line/column;
wenzelm
parents: 48775
diff changeset
   103
    var (line, column) = pos
6f3ccfa7818d more precise counting of line/column;
wenzelm
parents: 48775
diff changeset
   104
    for (sym <- iterator(text)) {
6f3ccfa7818d more precise counting of line/column;
wenzelm
parents: 48775
diff changeset
   105
      if (is_physical_newline(sym)) { line += 1; column = 1 }
6f3ccfa7818d more precise counting of line/column;
wenzelm
parents: 48775
diff changeset
   106
      else column += 1
6f3ccfa7818d more precise counting of line/column;
wenzelm
parents: 48775
diff changeset
   107
    }
6f3ccfa7818d more precise counting of line/column;
wenzelm
parents: 48775
diff changeset
   108
    (line, column)
6f3ccfa7818d more precise counting of line/column;
wenzelm
parents: 48775
diff changeset
   109
  }
6f3ccfa7818d more precise counting of line/column;
wenzelm
parents: 48775
diff changeset
   110
33998
fc56cfc6906e added elements: Interator;
wenzelm
parents: 31929
diff changeset
   111
fc56cfc6906e added elements: Interator;
wenzelm
parents: 31929
diff changeset
   112
  /* decoding offsets */
fc56cfc6906e added elements: Interator;
wenzelm
parents: 31929
diff changeset
   113
52507
27925b58d6bd tuned signature;
wenzelm
parents: 50564
diff changeset
   114
  object Index
27925b58d6bd tuned signature;
wenzelm
parents: 50564
diff changeset
   115
  {
27925b58d6bd tuned signature;
wenzelm
parents: 50564
diff changeset
   116
    def apply(text: CharSequence): Index = new Index(text)
27925b58d6bd tuned signature;
wenzelm
parents: 50564
diff changeset
   117
  }
27925b58d6bd tuned signature;
wenzelm
parents: 50564
diff changeset
   118
27925b58d6bd tuned signature;
wenzelm
parents: 50564
diff changeset
   119
  final class Index private(text: CharSequence)
31929
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   120
  {
43714
3749d1e6dde9 tuned signature;
wenzelm
parents: 43696
diff changeset
   121
    sealed case class Entry(chr: Int, sym: Int)
31929
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   122
    val index: Array[Entry] =
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   123
    {
34137
6cc9a0cbaf55 refined some Symbol operations/signatures;
wenzelm
parents: 34134
diff changeset
   124
      val matcher = new Matcher(text)
31929
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   125
      val buf = new mutable.ArrayBuffer[Entry]
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   126
      var chr = 0
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   127
      var sym = 0
33998
fc56cfc6906e added elements: Interator;
wenzelm
parents: 31929
diff changeset
   128
      while (chr < text.length) {
34137
6cc9a0cbaf55 refined some Symbol operations/signatures;
wenzelm
parents: 34134
diff changeset
   129
        val n = matcher(chr, text.length)
6cc9a0cbaf55 refined some Symbol operations/signatures;
wenzelm
parents: 34134
diff changeset
   130
        chr += n
31929
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   131
        sym += 1
34137
6cc9a0cbaf55 refined some Symbol operations/signatures;
wenzelm
parents: 34134
diff changeset
   132
        if (n > 1) buf += Entry(chr, sym)
31929
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   133
      }
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   134
      buf.toArray
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   135
    }
38479
e628da370072 more efficient Markup_Tree, based on branches sorted by quasi-order;
wenzelm
parents: 37556
diff changeset
   136
    def decode(sym1: Int): Int =
31929
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   137
    {
38479
e628da370072 more efficient Markup_Tree, based on branches sorted by quasi-order;
wenzelm
parents: 37556
diff changeset
   138
      val sym = sym1 - 1
31929
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   139
      val end = index.length
48922
6f3ccfa7818d more precise counting of line/column;
wenzelm
parents: 48775
diff changeset
   140
      @tailrec def bisect(a: Int, b: Int): Int =
31929
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   141
      {
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   142
        if (a < b) {
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   143
          val c = (a + b) / 2
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   144
          if (sym < index(c).sym) bisect(a, c)
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   145
          else if (c + 1 == end || sym < index(c + 1).sym) c
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   146
          else bisect(c + 1, b)
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   147
        }
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   148
        else -1
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   149
      }
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   150
      val i = bisect(0, end)
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   151
      if (i < 0) sym
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   152
      else index(i).chr + sym - index(i).sym
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   153
    }
38479
e628da370072 more efficient Markup_Tree, based on branches sorted by quasi-order;
wenzelm
parents: 37556
diff changeset
   154
    def decode(range: Text.Range): Text.Range = range.map(decode(_))
31929
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   155
  }
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   156
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   157
33998
fc56cfc6906e added elements: Interator;
wenzelm
parents: 31929
diff changeset
   158
  /* recoding text */
27937
fdf77e7be01a more robust pattern: look at longer matches first, added catch-all case;
wenzelm
parents: 27935
diff changeset
   159
31522
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   160
  private class Recoder(list: List[(String, String)])
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   161
  {
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   162
    private val (min, max) =
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   163
    {
27937
fdf77e7be01a more robust pattern: look at longer matches first, added catch-all case;
wenzelm
parents: 27935
diff changeset
   164
      var min = '\uffff'
fdf77e7be01a more robust pattern: look at longer matches first, added catch-all case;
wenzelm
parents: 27935
diff changeset
   165
      var max = '\u0000'
fdf77e7be01a more robust pattern: look at longer matches first, added catch-all case;
wenzelm
parents: 27935
diff changeset
   166
      for ((x, _) <- list) {
fdf77e7be01a more robust pattern: look at longer matches first, added catch-all case;
wenzelm
parents: 27935
diff changeset
   167
        val c = x(0)
fdf77e7be01a more robust pattern: look at longer matches first, added catch-all case;
wenzelm
parents: 27935
diff changeset
   168
        if (c < min) min = c
fdf77e7be01a more robust pattern: look at longer matches first, added catch-all case;
wenzelm
parents: 27935
diff changeset
   169
        if (c > max) max = c
fdf77e7be01a more robust pattern: look at longer matches first, added catch-all case;
wenzelm
parents: 27935
diff changeset
   170
      }
fdf77e7be01a more robust pattern: look at longer matches first, added catch-all case;
wenzelm
parents: 27935
diff changeset
   171
      (min, max)
fdf77e7be01a more robust pattern: look at longer matches first, added catch-all case;
wenzelm
parents: 27935
diff changeset
   172
    }
40443
41c32616298c explicitly check uniqueness of symbol recoding;
wenzelm
parents: 38877
diff changeset
   173
    private val table =
41c32616298c explicitly check uniqueness of symbol recoding;
wenzelm
parents: 38877
diff changeset
   174
    {
41c32616298c explicitly check uniqueness of symbol recoding;
wenzelm
parents: 38877
diff changeset
   175
      var tab = Map[String, String]()
41c32616298c explicitly check uniqueness of symbol recoding;
wenzelm
parents: 38877
diff changeset
   176
      for ((x, y) <- list) {
41c32616298c explicitly check uniqueness of symbol recoding;
wenzelm
parents: 38877
diff changeset
   177
        tab.get(x) match {
41c32616298c explicitly check uniqueness of symbol recoding;
wenzelm
parents: 38877
diff changeset
   178
          case None => tab += (x -> y)
41c32616298c explicitly check uniqueness of symbol recoding;
wenzelm
parents: 38877
diff changeset
   179
          case Some(z) =>
44181
wenzelm
parents: 43714
diff changeset
   180
            error("Duplicate mapping of " + quote(x) + " to " + quote(y) + " vs. " + quote(z))
40443
41c32616298c explicitly check uniqueness of symbol recoding;
wenzelm
parents: 38877
diff changeset
   181
        }
41c32616298c explicitly check uniqueness of symbol recoding;
wenzelm
parents: 38877
diff changeset
   182
      }
41c32616298c explicitly check uniqueness of symbol recoding;
wenzelm
parents: 38877
diff changeset
   183
      tab
41c32616298c explicitly check uniqueness of symbol recoding;
wenzelm
parents: 38877
diff changeset
   184
    }
31522
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   185
    def recode(text: String): String =
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   186
    {
27937
fdf77e7be01a more robust pattern: look at longer matches first, added catch-all case;
wenzelm
parents: 27935
diff changeset
   187
      val len = text.length
48775
92ceb058391f simplified symbol matching;
wenzelm
parents: 48774
diff changeset
   188
      val matcher = symbol_total.pattern.matcher(text)
27937
fdf77e7be01a more robust pattern: look at longer matches first, added catch-all case;
wenzelm
parents: 27935
diff changeset
   189
      val result = new StringBuilder(len)
fdf77e7be01a more robust pattern: look at longer matches first, added catch-all case;
wenzelm
parents: 27935
diff changeset
   190
      var i = 0
fdf77e7be01a more robust pattern: look at longer matches first, added catch-all case;
wenzelm
parents: 27935
diff changeset
   191
      while (i < len) {
fdf77e7be01a more robust pattern: look at longer matches first, added catch-all case;
wenzelm
parents: 27935
diff changeset
   192
        val c = text(i)
fdf77e7be01a more robust pattern: look at longer matches first, added catch-all case;
wenzelm
parents: 27935
diff changeset
   193
        if (min <= c && c <= max) {
31929
ecfc667cac53 is_open: surrogate sequence is High..Low;
wenzelm
parents: 31651
diff changeset
   194
          matcher.region(i, len).lookingAt
27938
3d5b12f23f15 recode: proper result for unmatched symbols;
wenzelm
parents: 27937
diff changeset
   195
          val x = matcher.group
52888
wenzelm
parents: 52616
diff changeset
   196
          result.append(table.getOrElse(x, x))
27937
fdf77e7be01a more robust pattern: look at longer matches first, added catch-all case;
wenzelm
parents: 27935
diff changeset
   197
          i = matcher.end
fdf77e7be01a more robust pattern: look at longer matches first, added catch-all case;
wenzelm
parents: 27935
diff changeset
   198
        }
fdf77e7be01a more robust pattern: look at longer matches first, added catch-all case;
wenzelm
parents: 27935
diff changeset
   199
        else { result.append(c); i += 1 }
fdf77e7be01a more robust pattern: look at longer matches first, added catch-all case;
wenzelm
parents: 27935
diff changeset
   200
      }
fdf77e7be01a more robust pattern: look at longer matches first, added catch-all case;
wenzelm
parents: 27935
diff changeset
   201
      result.toString
fdf77e7be01a more robust pattern: look at longer matches first, added catch-all case;
wenzelm
parents: 27935
diff changeset
   202
    }
fdf77e7be01a more robust pattern: look at longer matches first, added catch-all case;
wenzelm
parents: 27935
diff changeset
   203
  }
27924
8dd8b564faf5 tuned comments;
wenzelm
parents: 27923
diff changeset
   204
27918
85942d2036a0 reading symbol interpretation tables;
wenzelm
parents: 27905
diff changeset
   205
27923
7ebe9d38743a use scala.collection.jcl.HashMap, which seems to be more efficient;
wenzelm
parents: 27918
diff changeset
   206
43695
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   207
  /** symbol interpretation **/
27927
eb624bb54bc6 tuned Recoder;
wenzelm
parents: 27926
diff changeset
   208
43695
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   209
  private lazy val symbols =
50564
c6fde2fc4217 allow to suppress ISABELLE_SYMBOLS for experiments;
wenzelm
parents: 50291
diff changeset
   210
    new Interpretation(File.try_read(Path.split(Isabelle_System.getenv("ISABELLE_SYMBOLS"))))
43695
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   211
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   212
  private class Interpretation(symbols_spec: String)
29569
f3f529b5d8fb more general init of Symbol.Interpretation, independent of IsabelleSystem instance;
wenzelm
parents: 29174
diff changeset
   213
  {
31522
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   214
    /* read symbols */
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   215
50136
a96bd08258a2 support for symbol groups, retaining original order of declarations;
wenzelm
parents: 48922
diff changeset
   216
    private val No_Decl = new Regex("""(?xs) ^\s* (?: \#.* )? $ """)
a96bd08258a2 support for symbol groups, retaining original order of declarations;
wenzelm
parents: 48922
diff changeset
   217
    private val Key = new Regex("""(?xs) (.+): """)
31522
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   218
53316
c3e549e0d3c7 allow multiple symbol properties, notably groups and abbrevs;
wenzelm
parents: 53021
diff changeset
   219
    private def read_decl(decl: String): (Symbol, Properties.T) =
31522
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   220
    {
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   221
      def err() = error("Bad symbol declaration: " + decl)
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   222
53316
c3e549e0d3c7 allow multiple symbol properties, notably groups and abbrevs;
wenzelm
parents: 53021
diff changeset
   223
      def read_props(props: List[String]): Properties.T =
31522
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   224
      {
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   225
        props match {
53316
c3e549e0d3c7 allow multiple symbol properties, notably groups and abbrevs;
wenzelm
parents: 53021
diff changeset
   226
          case Nil => Nil
31522
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   227
          case _ :: Nil => err()
53316
c3e549e0d3c7 allow multiple symbol properties, notably groups and abbrevs;
wenzelm
parents: 53021
diff changeset
   228
          case Key(x) :: y :: rest => (x -> y) :: read_props(rest)
31522
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   229
          case _ => err()
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   230
        }
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   231
      }
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   232
      decl.split("\\s+").toList match {
53316
c3e549e0d3c7 allow multiple symbol properties, notably groups and abbrevs;
wenzelm
parents: 53021
diff changeset
   233
        case sym :: props if sym.length > 1 && !is_malformed(sym) =>
c3e549e0d3c7 allow multiple symbol properties, notably groups and abbrevs;
wenzelm
parents: 53021
diff changeset
   234
          (sym, read_props(props))
34193
d3358b909c40 some sanity checks for symbol interpretation;
wenzelm
parents: 34138
diff changeset
   235
        case _ => err()
31522
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   236
      }
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   237
    }
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   238
53316
c3e549e0d3c7 allow multiple symbol properties, notably groups and abbrevs;
wenzelm
parents: 53021
diff changeset
   239
    private val symbols: List[(Symbol, Properties.T)] =
c3e549e0d3c7 allow multiple symbol properties, notably groups and abbrevs;
wenzelm
parents: 53021
diff changeset
   240
      (((List.empty[(Symbol, Properties.T)], Set.empty[Symbol]) /:
50136
a96bd08258a2 support for symbol groups, retaining original order of declarations;
wenzelm
parents: 48922
diff changeset
   241
          split_lines(symbols_spec).reverse)
a96bd08258a2 support for symbol groups, retaining original order of declarations;
wenzelm
parents: 48922
diff changeset
   242
        { case (res, No_Decl()) => res
a96bd08258a2 support for symbol groups, retaining original order of declarations;
wenzelm
parents: 48922
diff changeset
   243
          case ((list, known), decl) =>
a96bd08258a2 support for symbol groups, retaining original order of declarations;
wenzelm
parents: 48922
diff changeset
   244
            val (sym, props) = read_decl(decl)
a96bd08258a2 support for symbol groups, retaining original order of declarations;
wenzelm
parents: 48922
diff changeset
   245
            if (known(sym)) (list, known)
a96bd08258a2 support for symbol groups, retaining original order of declarations;
wenzelm
parents: 48922
diff changeset
   246
            else ((sym, props) :: list, known + sym)
a96bd08258a2 support for symbol groups, retaining original order of declarations;
wenzelm
parents: 48922
diff changeset
   247
        })._1
31522
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   248
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   249
31651
7d6a518b5a2b added names, abbrevs;
wenzelm
parents: 31548
diff changeset
   250
    /* misc properties */
7d6a518b5a2b added names, abbrevs;
wenzelm
parents: 31548
diff changeset
   251
43696
58bb7ca5c7a2 explicit indication of type Symbol.Symbol;
wenzelm
parents: 43695
diff changeset
   252
    val names: Map[Symbol, String] =
34134
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   253
    {
43456
8a6de1a6e1dc names for control symbols without "^", which is relevant for completion;
wenzelm
parents: 43455
diff changeset
   254
      val name = new Regex("""\\<\^?([A-Za-z][A-Za-z0-9_']*)>""")
31651
7d6a518b5a2b added names, abbrevs;
wenzelm
parents: 31548
diff changeset
   255
      Map((for ((sym @ name(a), _) <- symbols) yield (sym -> a)): _*)
7d6a518b5a2b added names, abbrevs;
wenzelm
parents: 31548
diff changeset
   256
    }
7d6a518b5a2b added names, abbrevs;
wenzelm
parents: 31548
diff changeset
   257
50136
a96bd08258a2 support for symbol groups, retaining original order of declarations;
wenzelm
parents: 48922
diff changeset
   258
    val groups: List[(String, List[Symbol])] =
53316
c3e549e0d3c7 allow multiple symbol properties, notably groups and abbrevs;
wenzelm
parents: 53021
diff changeset
   259
      symbols.map({ case (sym, props) =>
c3e549e0d3c7 allow multiple symbol properties, notably groups and abbrevs;
wenzelm
parents: 53021
diff changeset
   260
        val gs = for (("group", g) <- props) yield g
c3e549e0d3c7 allow multiple symbol properties, notably groups and abbrevs;
wenzelm
parents: 53021
diff changeset
   261
        if (gs.isEmpty) List(sym -> "unsorted") else gs.map(sym -> _)
c3e549e0d3c7 allow multiple symbol properties, notably groups and abbrevs;
wenzelm
parents: 53021
diff changeset
   262
      }).flatten
50136
a96bd08258a2 support for symbol groups, retaining original order of declarations;
wenzelm
parents: 48922
diff changeset
   263
        .groupBy(_._2).toList.map({ case (group, list) => (group, list.map(_._1)) })
a96bd08258a2 support for symbol groups, retaining original order of declarations;
wenzelm
parents: 48922
diff changeset
   264
        .sortBy(_._1)
a96bd08258a2 support for symbol groups, retaining original order of declarations;
wenzelm
parents: 48922
diff changeset
   265
53316
c3e549e0d3c7 allow multiple symbol properties, notably groups and abbrevs;
wenzelm
parents: 53021
diff changeset
   266
    val abbrevs: Multi_Map[Symbol, String] =
c3e549e0d3c7 allow multiple symbol properties, notably groups and abbrevs;
wenzelm
parents: 53021
diff changeset
   267
      Multi_Map((
c3e549e0d3c7 allow multiple symbol properties, notably groups and abbrevs;
wenzelm
parents: 53021
diff changeset
   268
        for {
c3e549e0d3c7 allow multiple symbol properties, notably groups and abbrevs;
wenzelm
parents: 53021
diff changeset
   269
          (sym, props) <- symbols
c3e549e0d3c7 allow multiple symbol properties, notably groups and abbrevs;
wenzelm
parents: 53021
diff changeset
   270
          ("abbrev", a) <- props.reverse
c3e549e0d3c7 allow multiple symbol properties, notably groups and abbrevs;
wenzelm
parents: 53021
diff changeset
   271
        } yield (sym -> a)): _*)
43488
39035276927c Symbol.is_ctrl: handle decoded version as well;
wenzelm
parents: 43487
diff changeset
   272
39035276927c Symbol.is_ctrl: handle decoded version as well;
wenzelm
parents: 43487
diff changeset
   273
43490
5e6f76cacb93 more uniform treatment of recode_set/recode_map;
wenzelm
parents: 43489
diff changeset
   274
    /* recoding */
31522
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   275
53316
c3e549e0d3c7 allow multiple symbol properties, notably groups and abbrevs;
wenzelm
parents: 53021
diff changeset
   276
    private val Code = new Properties.String("code")
31522
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   277
    private val (decoder, encoder) =
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   278
    {
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   279
      val mapping =
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   280
        for {
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   281
          (sym, props) <- symbols
46997
395b7277ed76 misc tuning to accomodate scala-2.10.0-M2;
wenzelm
parents: 44992
diff changeset
   282
          code =
53316
c3e549e0d3c7 allow multiple symbol properties, notably groups and abbrevs;
wenzelm
parents: 53021
diff changeset
   283
            props match {
c3e549e0d3c7 allow multiple symbol properties, notably groups and abbrevs;
wenzelm
parents: 53021
diff changeset
   284
              case Code(s) =>
c3e549e0d3c7 allow multiple symbol properties, notably groups and abbrevs;
wenzelm
parents: 53021
diff changeset
   285
                try { Integer.decode(s).intValue }
c3e549e0d3c7 allow multiple symbol properties, notably groups and abbrevs;
wenzelm
parents: 53021
diff changeset
   286
                catch { case _: NumberFormatException => error("Bad code for symbol " + sym) }
c3e549e0d3c7 allow multiple symbol properties, notably groups and abbrevs;
wenzelm
parents: 53021
diff changeset
   287
              case _ => error("Missing code for symbol " + sym)
31522
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   288
            }
46997
395b7277ed76 misc tuning to accomodate scala-2.10.0-M2;
wenzelm
parents: 44992
diff changeset
   289
          ch = new String(Character.toChars(code))
34193
d3358b909c40 some sanity checks for symbol interpretation;
wenzelm
parents: 34138
diff changeset
   290
        } yield {
d3358b909c40 some sanity checks for symbol interpretation;
wenzelm
parents: 34138
diff changeset
   291
          if (code < 128) error("Illegal ASCII code for symbol " + sym)
d3358b909c40 some sanity checks for symbol interpretation;
wenzelm
parents: 34138
diff changeset
   292
          else (sym, ch)
d3358b909c40 some sanity checks for symbol interpretation;
wenzelm
parents: 34138
diff changeset
   293
        }
31545
5f1f0a20af4d discontinued escaped symbols such as \\<forall> -- only one backslash should be used;
wenzelm
parents: 31523
diff changeset
   294
      (new Recoder(mapping),
31548
wenzelm
parents: 31545
diff changeset
   295
       new Recoder(mapping map { case (x, y) => (y, x) }))
31522
0466cb17064f more native Scala style;
wenzelm
parents: 29569
diff changeset
   296
    }
27918
85942d2036a0 reading symbol interpretation tables;
wenzelm
parents: 27905
diff changeset
   297
34098
2b9cdf23c188 tuned signature;
wenzelm
parents: 34001
diff changeset
   298
    def decode(text: String): String = decoder.recode(text)
2b9cdf23c188 tuned signature;
wenzelm
parents: 34001
diff changeset
   299
    def encode(text: String): String = encoder.recode(text)
34134
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   300
43490
5e6f76cacb93 more uniform treatment of recode_set/recode_map;
wenzelm
parents: 43489
diff changeset
   301
    private def recode_set(elems: String*): Set[String] =
5e6f76cacb93 more uniform treatment of recode_set/recode_map;
wenzelm
parents: 43489
diff changeset
   302
    {
5e6f76cacb93 more uniform treatment of recode_set/recode_map;
wenzelm
parents: 43489
diff changeset
   303
      val content = elems.toList
5e6f76cacb93 more uniform treatment of recode_set/recode_map;
wenzelm
parents: 43489
diff changeset
   304
      Set((content ::: content.map(decode)): _*)
5e6f76cacb93 more uniform treatment of recode_set/recode_map;
wenzelm
parents: 43489
diff changeset
   305
    }
5e6f76cacb93 more uniform treatment of recode_set/recode_map;
wenzelm
parents: 43489
diff changeset
   306
5e6f76cacb93 more uniform treatment of recode_set/recode_map;
wenzelm
parents: 43489
diff changeset
   307
    private def recode_map[A](elems: (String, A)*): Map[String, A] =
5e6f76cacb93 more uniform treatment of recode_set/recode_map;
wenzelm
parents: 43489
diff changeset
   308
    {
5e6f76cacb93 more uniform treatment of recode_set/recode_map;
wenzelm
parents: 43489
diff changeset
   309
      val content = elems.toList
5e6f76cacb93 more uniform treatment of recode_set/recode_map;
wenzelm
parents: 43489
diff changeset
   310
      Map((content ::: content.map({ case (sym, a) => (decode(sym), a) })): _*)
5e6f76cacb93 more uniform treatment of recode_set/recode_map;
wenzelm
parents: 43489
diff changeset
   311
    }
5e6f76cacb93 more uniform treatment of recode_set/recode_map;
wenzelm
parents: 43489
diff changeset
   312
5e6f76cacb93 more uniform treatment of recode_set/recode_map;
wenzelm
parents: 43489
diff changeset
   313
5e6f76cacb93 more uniform treatment of recode_set/recode_map;
wenzelm
parents: 43489
diff changeset
   314
    /* user fonts */
5e6f76cacb93 more uniform treatment of recode_set/recode_map;
wenzelm
parents: 43489
diff changeset
   315
53316
c3e549e0d3c7 allow multiple symbol properties, notably groups and abbrevs;
wenzelm
parents: 53021
diff changeset
   316
    private val Font = new Properties.String("font")
43696
58bb7ca5c7a2 explicit indication of type Symbol.Symbol;
wenzelm
parents: 43695
diff changeset
   317
    val fonts: Map[Symbol, String] =
53316
c3e549e0d3c7 allow multiple symbol properties, notably groups and abbrevs;
wenzelm
parents: 53021
diff changeset
   318
      recode_map((for ((sym, Font(font)) <- symbols) yield (sym -> font)): _*)
43490
5e6f76cacb93 more uniform treatment of recode_set/recode_map;
wenzelm
parents: 43489
diff changeset
   319
5e6f76cacb93 more uniform treatment of recode_set/recode_map;
wenzelm
parents: 43489
diff changeset
   320
    val font_names: List[String] = Set(fonts.toList.map(_._2): _*).toList
5e6f76cacb93 more uniform treatment of recode_set/recode_map;
wenzelm
parents: 43489
diff changeset
   321
    val font_index: Map[String, Int] = Map((font_names zip (0 until font_names.length).toList): _*)
5e6f76cacb93 more uniform treatment of recode_set/recode_map;
wenzelm
parents: 43489
diff changeset
   322
34134
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   323
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   324
    /* classification */
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   325
43695
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   326
    val letters = recode_set(
34134
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   327
      "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M",
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   328
      "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z",
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   329
      "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   330
      "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z",
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   331
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   332
      "\\<A>", "\\<B>", "\\<C>", "\\<D>", "\\<E>", "\\<F>", "\\<G>",
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   333
      "\\<H>", "\\<I>", "\\<J>", "\\<K>", "\\<L>", "\\<M>", "\\<N>",
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   334
      "\\<O>", "\\<P>", "\\<Q>", "\\<R>", "\\<S>", "\\<T>", "\\<U>",
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   335
      "\\<V>", "\\<W>", "\\<X>", "\\<Y>", "\\<Z>", "\\<a>", "\\<b>",
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   336
      "\\<c>", "\\<d>", "\\<e>", "\\<f>", "\\<g>", "\\<h>", "\\<i>",
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   337
      "\\<j>", "\\<k>", "\\<l>", "\\<m>", "\\<n>", "\\<o>", "\\<p>",
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   338
      "\\<q>", "\\<r>", "\\<s>", "\\<t>", "\\<u>", "\\<v>", "\\<w>",
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   339
      "\\<x>", "\\<y>", "\\<z>",
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   340
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   341
      "\\<AA>", "\\<BB>", "\\<CC>", "\\<DD>", "\\<EE>", "\\<FF>",
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   342
      "\\<GG>", "\\<HH>", "\\<II>", "\\<JJ>", "\\<KK>", "\\<LL>",
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   343
      "\\<MM>", "\\<NN>", "\\<OO>", "\\<PP>", "\\<QQ>", "\\<RR>",
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   344
      "\\<SS>", "\\<TT>", "\\<UU>", "\\<VV>", "\\<WW>", "\\<XX>",
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   345
      "\\<YY>", "\\<ZZ>", "\\<aa>", "\\<bb>", "\\<cc>", "\\<dd>",
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   346
      "\\<ee>", "\\<ff>", "\\<gg>", "\\<hh>", "\\<ii>", "\\<jj>",
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   347
      "\\<kk>", "\\<ll>", "\\<mm>", "\\<nn>", "\\<oo>", "\\<pp>",
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   348
      "\\<qq>", "\\<rr>", "\\<ss>", "\\<tt>", "\\<uu>", "\\<vv>",
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   349
      "\\<ww>", "\\<xx>", "\\<yy>", "\\<zz>",
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   350
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   351
      "\\<alpha>", "\\<beta>", "\\<gamma>", "\\<delta>", "\\<epsilon>",
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   352
      "\\<zeta>", "\\<eta>", "\\<theta>", "\\<iota>", "\\<kappa>",
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   353
      "\\<mu>", "\\<nu>", "\\<xi>", "\\<pi>", "\\<rho>", "\\<sigma>",
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   354
      "\\<tau>", "\\<upsilon>", "\\<phi>", "\\<chi>", "\\<psi>",
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   355
      "\\<omega>", "\\<Gamma>", "\\<Delta>", "\\<Theta>", "\\<Lambda>",
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   356
      "\\<Xi>", "\\<Pi>", "\\<Sigma>", "\\<Upsilon>", "\\<Phi>",
52616
3ac2878764f9 more robust identifier syntax: sub/superscript counts as modifier of LETDIG part instead of LETTER, both isub/isup and sub/sup are allowed;
wenzelm
parents: 52507
diff changeset
   357
      "\\<Psi>", "\\<Omega>")
34134
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   358
50233
eef21a0726f1 removed remains of Oheimb's double-space (cf. 0a5af667dc75);
wenzelm
parents: 50188
diff changeset
   359
    val blanks = recode_set(" ", "\t", "\n", "\u000B", "\f", "\r", "\r\n", "\\<^newline>")
34138
4008c2f5a46e refined some Symbol operations/signatures;
wenzelm
parents: 34137
diff changeset
   360
43695
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   361
    val sym_chars =
34138
4008c2f5a46e refined some Symbol operations/signatures;
wenzelm
parents: 34137
diff changeset
   362
      Set("!", "#", "$", "%", "&", "*", "+", "-", "/", "<", "=", ">", "?", "@", "^", "_", "|", "~")
34134
d8d9df8407f6 added symbol classification;
wenzelm
parents: 34098
diff changeset
   363
44992
aa34d2d049ce refined Symbol.is_symbolic -- cover recoded versions as well;
wenzelm
parents: 44949
diff changeset
   364
    val symbolic = recode_set((for { (sym, _) <- symbols; if raw_symbolic(sym) } yield sym): _*)
aa34d2d049ce refined Symbol.is_symbolic -- cover recoded versions as well;
wenzelm
parents: 44949
diff changeset
   365
43455
4b4b93672f15 some unicode chars for special control symbols;
wenzelm
parents: 43447
diff changeset
   366
43488
39035276927c Symbol.is_ctrl: handle decoded version as well;
wenzelm
parents: 43487
diff changeset
   367
    /* control symbols */
39035276927c Symbol.is_ctrl: handle decoded version as well;
wenzelm
parents: 43487
diff changeset
   368
43696
58bb7ca5c7a2 explicit indication of type Symbol.Symbol;
wenzelm
parents: 43695
diff changeset
   369
    val ctrl_decoded: Set[Symbol] =
43488
39035276927c Symbol.is_ctrl: handle decoded version as well;
wenzelm
parents: 43487
diff changeset
   370
      Set((for ((sym, _) <- symbols if sym.startsWith("\\<^")) yield decode(sym)): _*)
39035276927c Symbol.is_ctrl: handle decoded version as well;
wenzelm
parents: 43487
diff changeset
   371
44238
36120feb70ed some convenience actions/shortcuts for control symbols;
wenzelm
parents: 44181
diff changeset
   372
    val sub_decoded = decode("\\<^sub>")
36120feb70ed some convenience actions/shortcuts for control symbols;
wenzelm
parents: 44181
diff changeset
   373
    val sup_decoded = decode("\\<^sup>")
43511
d138e7482a1b clarified decoded control symbols;
wenzelm
parents: 43490
diff changeset
   374
    val bsub_decoded = decode("\\<^bsub>")
d138e7482a1b clarified decoded control symbols;
wenzelm
parents: 43490
diff changeset
   375
    val esub_decoded = decode("\\<^esub>")
d138e7482a1b clarified decoded control symbols;
wenzelm
parents: 43490
diff changeset
   376
    val bsup_decoded = decode("\\<^bsup>")
d138e7482a1b clarified decoded control symbols;
wenzelm
parents: 43490
diff changeset
   377
    val esup_decoded = decode("\\<^esup>")
44238
36120feb70ed some convenience actions/shortcuts for control symbols;
wenzelm
parents: 44181
diff changeset
   378
    val bold_decoded = decode("\\<^bold>")
27918
85942d2036a0 reading symbol interpretation tables;
wenzelm
parents: 27905
diff changeset
   379
  }
43695
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   380
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   381
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   382
  /* tables */
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   383
43696
58bb7ca5c7a2 explicit indication of type Symbol.Symbol;
wenzelm
parents: 43695
diff changeset
   384
  def names: Map[Symbol, String] = symbols.names
50136
a96bd08258a2 support for symbol groups, retaining original order of declarations;
wenzelm
parents: 48922
diff changeset
   385
  def groups: List[(String, List[Symbol])] = symbols.groups
53316
c3e549e0d3c7 allow multiple symbol properties, notably groups and abbrevs;
wenzelm
parents: 53021
diff changeset
   386
  def abbrevs: Multi_Map[Symbol, String] = symbols.abbrevs
43695
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   387
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   388
  def decode(text: String): String = symbols.decode(text)
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   389
  def encode(text: String): String = symbols.encode(text)
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   390
53337
b3817a0e3211 sort items according to persistent history of frequency of use;
wenzelm
parents: 53316
diff changeset
   391
  def decode_string: XML.Decode.T[String] = (x => decode(XML.Decode.string(x)))
b3817a0e3211 sort items according to persistent history of frequency of use;
wenzelm
parents: 53316
diff changeset
   392
  def encode_string: XML.Encode.T[String] = (x => XML.Encode.string(encode(x)))
b3817a0e3211 sort items according to persistent history of frequency of use;
wenzelm
parents: 53316
diff changeset
   393
50291
674893679352 prefer Symbol.decode_strict in batch mode, to avoid files with spurious Unicode symbols that clash with Isabelle symbol interpretation;
wenzelm
parents: 50238
diff changeset
   394
  def decode_strict(text: String): String =
674893679352 prefer Symbol.decode_strict in batch mode, to avoid files with spurious Unicode symbols that clash with Isabelle symbol interpretation;
wenzelm
parents: 50238
diff changeset
   395
  {
674893679352 prefer Symbol.decode_strict in batch mode, to avoid files with spurious Unicode symbols that clash with Isabelle symbol interpretation;
wenzelm
parents: 50238
diff changeset
   396
    val decoded = decode(text)
674893679352 prefer Symbol.decode_strict in batch mode, to avoid files with spurious Unicode symbols that clash with Isabelle symbol interpretation;
wenzelm
parents: 50238
diff changeset
   397
    if (encode(decoded) == text) decoded
674893679352 prefer Symbol.decode_strict in batch mode, to avoid files with spurious Unicode symbols that clash with Isabelle symbol interpretation;
wenzelm
parents: 50238
diff changeset
   398
    else {
674893679352 prefer Symbol.decode_strict in batch mode, to avoid files with spurious Unicode symbols that clash with Isabelle symbol interpretation;
wenzelm
parents: 50238
diff changeset
   399
      val bad = new mutable.ListBuffer[Symbol]
674893679352 prefer Symbol.decode_strict in batch mode, to avoid files with spurious Unicode symbols that clash with Isabelle symbol interpretation;
wenzelm
parents: 50238
diff changeset
   400
      for (s <- iterator(text) if encode(decode(s)) != s && !bad.contains(s))
674893679352 prefer Symbol.decode_strict in batch mode, to avoid files with spurious Unicode symbols that clash with Isabelle symbol interpretation;
wenzelm
parents: 50238
diff changeset
   401
        bad += s
674893679352 prefer Symbol.decode_strict in batch mode, to avoid files with spurious Unicode symbols that clash with Isabelle symbol interpretation;
wenzelm
parents: 50238
diff changeset
   402
      error("Bad Unicode symbols in text: " + commas_quote(bad))
674893679352 prefer Symbol.decode_strict in batch mode, to avoid files with spurious Unicode symbols that clash with Isabelle symbol interpretation;
wenzelm
parents: 50238
diff changeset
   403
    }
674893679352 prefer Symbol.decode_strict in batch mode, to avoid files with spurious Unicode symbols that clash with Isabelle symbol interpretation;
wenzelm
parents: 50238
diff changeset
   404
  }
674893679352 prefer Symbol.decode_strict in batch mode, to avoid files with spurious Unicode symbols that clash with Isabelle symbol interpretation;
wenzelm
parents: 50238
diff changeset
   405
43696
58bb7ca5c7a2 explicit indication of type Symbol.Symbol;
wenzelm
parents: 43695
diff changeset
   406
  def fonts: Map[Symbol, String] = symbols.fonts
43695
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   407
  def font_names: List[String] = symbols.font_names
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   408
  def font_index: Map[String, Int] = symbols.font_index
43696
58bb7ca5c7a2 explicit indication of type Symbol.Symbol;
wenzelm
parents: 43695
diff changeset
   409
  def lookup_font(sym: Symbol): Option[Int] = symbols.fonts.get(sym).map(font_index(_))
43695
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   410
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   411
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   412
  /* classification */
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   413
43696
58bb7ca5c7a2 explicit indication of type Symbol.Symbol;
wenzelm
parents: 43695
diff changeset
   414
  def is_letter(sym: Symbol): Boolean = symbols.letters.contains(sym)
58bb7ca5c7a2 explicit indication of type Symbol.Symbol;
wenzelm
parents: 43695
diff changeset
   415
  def is_digit(sym: Symbol): Boolean = sym.length == 1 && '0' <= sym(0) && sym(0) <= '9'
58bb7ca5c7a2 explicit indication of type Symbol.Symbol;
wenzelm
parents: 43695
diff changeset
   416
  def is_quasi(sym: Symbol): Boolean = sym == "_" || sym == "'"
58bb7ca5c7a2 explicit indication of type Symbol.Symbol;
wenzelm
parents: 43695
diff changeset
   417
  def is_letdig(sym: Symbol): Boolean = is_letter(sym) || is_digit(sym) || is_quasi(sym)
58bb7ca5c7a2 explicit indication of type Symbol.Symbol;
wenzelm
parents: 43695
diff changeset
   418
  def is_blank(sym: Symbol): Boolean = symbols.blanks.contains(sym)
44992
aa34d2d049ce refined Symbol.is_symbolic -- cover recoded versions as well;
wenzelm
parents: 44949
diff changeset
   419
43696
58bb7ca5c7a2 explicit indication of type Symbol.Symbol;
wenzelm
parents: 43695
diff changeset
   420
  def is_symbolic_char(sym: Symbol): Boolean = symbols.sym_chars.contains(sym)
44992
aa34d2d049ce refined Symbol.is_symbolic -- cover recoded versions as well;
wenzelm
parents: 44949
diff changeset
   421
  def is_symbolic(sym: Symbol): Boolean = raw_symbolic(sym) || symbols.symbolic.contains(sym)
aa34d2d049ce refined Symbol.is_symbolic -- cover recoded versions as well;
wenzelm
parents: 44949
diff changeset
   422
aa34d2d049ce refined Symbol.is_symbolic -- cover recoded versions as well;
wenzelm
parents: 44949
diff changeset
   423
  private def raw_symbolic(sym: Symbol): Boolean =
43695
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   424
    sym.startsWith("\\<") && sym.endsWith(">") && !sym.startsWith("\\<^")
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   425
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   426
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   427
  /* control symbols */
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   428
43696
58bb7ca5c7a2 explicit indication of type Symbol.Symbol;
wenzelm
parents: 43695
diff changeset
   429
  def is_ctrl(sym: Symbol): Boolean =
43695
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   430
    sym.startsWith("\\<^") || symbols.ctrl_decoded.contains(sym)
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   431
43696
58bb7ca5c7a2 explicit indication of type Symbol.Symbol;
wenzelm
parents: 43695
diff changeset
   432
  def is_controllable(sym: Symbol): Boolean =
43695
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   433
    !is_blank(sym) && !is_ctrl(sym) && !is_malformed(sym)
5130dfe1b7be simplified Symbol based on lazy Symbol.Interpretation -- reduced odd "functorial style";
wenzelm
parents: 43675
diff changeset
   434
44238
36120feb70ed some convenience actions/shortcuts for control symbols;
wenzelm
parents: 44181
diff changeset
   435
  def sub_decoded: Symbol = symbols.sub_decoded
36120feb70ed some convenience actions/shortcuts for control symbols;
wenzelm
parents: 44181
diff changeset
   436
  def sup_decoded: Symbol = symbols.sup_decoded
36120feb70ed some convenience actions/shortcuts for control symbols;
wenzelm
parents: 44181
diff changeset
   437
  def bsub_decoded: Symbol = symbols.bsub_decoded
36120feb70ed some convenience actions/shortcuts for control symbols;
wenzelm
parents: 44181
diff changeset
   438
  def esub_decoded: Symbol = symbols.esub_decoded
36120feb70ed some convenience actions/shortcuts for control symbols;
wenzelm
parents: 44181
diff changeset
   439
  def bsup_decoded: Symbol = symbols.bsup_decoded
36120feb70ed some convenience actions/shortcuts for control symbols;
wenzelm
parents: 44181
diff changeset
   440
  def esup_decoded: Symbol = symbols.esup_decoded
36120feb70ed some convenience actions/shortcuts for control symbols;
wenzelm
parents: 44181
diff changeset
   441
  def bold_decoded: Symbol = symbols.bold_decoded
27901
28083e9f8d1d Basic support for Isabelle symbols.
wenzelm
parents:
diff changeset
   442
}