author | wenzelm |
Sun, 21 Jan 2024 14:05:14 +0100 | |
changeset 79510 | d8330439823a |
parent 79509 | e82448aacf48 |
child 80350 | 96843eb96493 |
permissions | -rw-r--r-- |
54439 | 1 |
/* Title: Pure/General/bytes.scala |
2 |
Author: Makarius |
|
3 |
||
4 |
Immutable byte vectors versus UTF8 strings. |
|
5 |
*/ |
|
6 |
||
7 |
package isabelle |
|
8 |
||
9 |
||
78855 | 10 |
import java.io.{ByteArrayInputStream, ByteArrayOutputStream, FileInputStream, FileOutputStream, |
11 |
InputStream, OutputStream, File => JFile} |
|
77711
25fd62cba347
clarified signature: more general operation Bytes.read_slice;
wenzelm
parents:
76361
diff
changeset
|
12 |
import java.nio.ByteBuffer |
25fd62cba347
clarified signature: more general operation Bytes.read_slice;
wenzelm
parents:
76361
diff
changeset
|
13 |
import java.nio.channels.FileChannel |
25fd62cba347
clarified signature: more general operation Bytes.read_slice;
wenzelm
parents:
76361
diff
changeset
|
14 |
import java.nio.file.StandardOpenOption |
77712
dd4bb80dbc3a
tuned performance: much faster low-level operation;
wenzelm
parents:
77711
diff
changeset
|
15 |
import java.util.Arrays |
76353 | 16 |
import org.tukaani.xz |
17 |
import com.github.luben.zstd |
|
54440 | 18 |
|
19 |
||
75393 | 20 |
object Bytes { |
54439 | 21 |
val empty: Bytes = new Bytes(Array[Byte](), 0, 0) |
22 |
||
75393 | 23 |
def apply(s: CharSequence): Bytes = { |
54439 | 24 |
val str = s.toString |
25 |
if (str.isEmpty) empty |
|
26 |
else { |
|
62527 | 27 |
val b = UTF8.bytes(str) |
54442
c39972ddd672
more specific Protocol_Output: empty message.body, main content via bytes/text;
wenzelm
parents:
54440
diff
changeset
|
28 |
new Bytes(b, 0, b.length) |
54439 | 29 |
} |
30 |
} |
|
54440 | 31 |
|
63779 | 32 |
def apply(a: Array[Byte]): Bytes = apply(a, 0, a.length) |
33 |
||
54442
c39972ddd672
more specific Protocol_Output: empty message.body, main content via bytes/text;
wenzelm
parents:
54440
diff
changeset
|
34 |
def apply(a: Array[Byte], offset: Int, length: Int): Bytes = |
c39972ddd672
more specific Protocol_Output: empty message.body, main content via bytes/text;
wenzelm
parents:
54440
diff
changeset
|
35 |
if (length == 0) empty |
c39972ddd672
more specific Protocol_Output: empty message.body, main content via bytes/text;
wenzelm
parents:
54440
diff
changeset
|
36 |
else { |
c39972ddd672
more specific Protocol_Output: empty message.body, main content via bytes/text;
wenzelm
parents:
54440
diff
changeset
|
37 |
val b = new Array[Byte](length) |
55618 | 38 |
System.arraycopy(a, offset, b, 0, length) |
54442
c39972ddd672
more specific Protocol_Output: empty message.body, main content via bytes/text;
wenzelm
parents:
54440
diff
changeset
|
39 |
new Bytes(b, 0, b.length) |
c39972ddd672
more specific Protocol_Output: empty message.body, main content via bytes/text;
wenzelm
parents:
54440
diff
changeset
|
40 |
} |
c39972ddd672
more specific Protocol_Output: empty message.body, main content via bytes/text;
wenzelm
parents:
54440
diff
changeset
|
41 |
|
69454 | 42 |
val newline: Bytes = apply("\n") |
54440 | 43 |
|
73576 | 44 |
|
45 |
/* base64 */ |
|
46 |
||
75587 | 47 |
def decode_base64(s: String): Bytes = { |
75620 | 48 |
val a = Base64.decode(s) |
68108 | 49 |
new Bytes(a, 0, a.length) |
50 |
} |
|
51 |
||
75579 | 52 |
|
54440 | 53 |
/* read */ |
54 |
||
78243 | 55 |
def read_stream(stream: InputStream, limit: Int = Int.MaxValue, hint: Int = 1024): Bytes = |
64004 | 56 |
if (limit == 0) empty |
57 |
else { |
|
78243 | 58 |
val out_size = (if (limit == Int.MaxValue) hint else limit) max 1024 |
73414 | 59 |
val out = new ByteArrayOutputStream(out_size) |
64005
f6e965cf1617
clarified magic values (see also java/io/BufferedInputStream.java);
wenzelm
parents:
64004
diff
changeset
|
60 |
val buf = new Array[Byte](8192) |
64004 | 61 |
var m = 0 |
54440 | 62 |
|
75709 | 63 |
while ({ |
73554 | 64 |
m = stream.read(buf, 0, buf.length min (limit - out.size)) |
64004 | 65 |
if (m != -1) out.write(buf, 0, m) |
75709 | 66 |
m != -1 && limit > out.size |
67 |
}) () |
|
64004 | 68 |
|
69 |
new Bytes(out.toByteArray, 0, out.size) |
|
54440 | 70 |
} |
64001
7ecb22be8f03
more general read_stream: return actual byte count;
wenzelm
parents:
63779
diff
changeset
|
71 |
|
79510
d8330439823a
clarified signature: explicit type isabelle.Url to avoid oddities of java.net.URL (e.g. its "equals" method);
wenzelm
parents:
79509
diff
changeset
|
72 |
def read_url(name: String): Bytes = using(Url(name).open_stream())(read_stream(_)) |
77717 | 73 |
|
78953 | 74 |
def read_file(path: Path, offset: Long = 0L, limit: Long = Long.MaxValue): Bytes = { |
78956 | 75 |
val length = File.size(path) |
77711
25fd62cba347
clarified signature: more general operation Bytes.read_slice;
wenzelm
parents:
76361
diff
changeset
|
76 |
val start = offset.max(0L) |
78953 | 77 |
val len = (length - start).max(0L).min(limit) |
78243 | 78 |
if (len > Int.MaxValue) error("Cannot read large file slice: " + Space.bytes(len).print) |
77711
25fd62cba347
clarified signature: more general operation Bytes.read_slice;
wenzelm
parents:
76361
diff
changeset
|
79 |
else if (len == 0L) empty |
25fd62cba347
clarified signature: more general operation Bytes.read_slice;
wenzelm
parents:
76361
diff
changeset
|
80 |
else { |
78953 | 81 |
using(FileChannel.open(path.java_path, StandardOpenOption.READ)) { java_path => |
77718 | 82 |
java_path.position(start) |
77711
25fd62cba347
clarified signature: more general operation Bytes.read_slice;
wenzelm
parents:
76361
diff
changeset
|
83 |
val n = len.toInt |
25fd62cba347
clarified signature: more general operation Bytes.read_slice;
wenzelm
parents:
76361
diff
changeset
|
84 |
val buf = ByteBuffer.allocate(n) |
25fd62cba347
clarified signature: more general operation Bytes.read_slice;
wenzelm
parents:
76361
diff
changeset
|
85 |
var i = 0 |
25fd62cba347
clarified signature: more general operation Bytes.read_slice;
wenzelm
parents:
76361
diff
changeset
|
86 |
var m = 0 |
25fd62cba347
clarified signature: more general operation Bytes.read_slice;
wenzelm
parents:
76361
diff
changeset
|
87 |
while ({ |
77718 | 88 |
m = java_path.read(buf) |
77711
25fd62cba347
clarified signature: more general operation Bytes.read_slice;
wenzelm
parents:
76361
diff
changeset
|
89 |
if (m != -1) i += m |
25fd62cba347
clarified signature: more general operation Bytes.read_slice;
wenzelm
parents:
76361
diff
changeset
|
90 |
m != -1 && n > i |
25fd62cba347
clarified signature: more general operation Bytes.read_slice;
wenzelm
parents:
76361
diff
changeset
|
91 |
}) () |
25fd62cba347
clarified signature: more general operation Bytes.read_slice;
wenzelm
parents:
76361
diff
changeset
|
92 |
new Bytes(buf.array, 0, i) |
25fd62cba347
clarified signature: more general operation Bytes.read_slice;
wenzelm
parents:
76361
diff
changeset
|
93 |
} |
25fd62cba347
clarified signature: more general operation Bytes.read_slice;
wenzelm
parents:
76361
diff
changeset
|
94 |
} |
25fd62cba347
clarified signature: more general operation Bytes.read_slice;
wenzelm
parents:
76361
diff
changeset
|
95 |
} |
25fd62cba347
clarified signature: more general operation Bytes.read_slice;
wenzelm
parents:
76361
diff
changeset
|
96 |
|
78953 | 97 |
def read(path: Path): Bytes = read_file(path) |
98 |
def read(file: JFile): Bytes = read_file(File.path(file)) |
|
77718 | 99 |
|
64229 | 100 |
|
101 |
/* write */ |
|
102 |
||
69393
ed0824ef337e
static type for Library.using: avoid Java 11 warnings on "Illegal reflective access";
wenzelm
parents:
69365
diff
changeset
|
103 |
def write(file: JFile, bytes: Bytes): Unit = |
ed0824ef337e
static type for Library.using: avoid Java 11 warnings on "Illegal reflective access";
wenzelm
parents:
69365
diff
changeset
|
104 |
using(new FileOutputStream(file))(bytes.write_stream(_)) |
64229 | 105 |
|
106 |
def write(path: Path, bytes: Bytes): Unit = write(path.file, bytes) |
|
78194 | 107 |
|
108 |
||
109 |
/* append */ |
|
110 |
||
111 |
def append(file: JFile, bytes: Bytes): Unit = |
|
112 |
using(new FileOutputStream(file, true))(bytes.write_stream(_)) |
|
113 |
||
114 |
def append(path: Path, bytes: Bytes): Unit = append(path.file, bytes) |
|
54439 | 115 |
} |
116 |
||
117 |
final class Bytes private( |
|
118 |
protected val bytes: Array[Byte], |
|
119 |
protected val offset: Int, |
|
75393 | 120 |
val length: Int) extends CharSequence { |
54439 | 121 |
/* equality */ |
122 |
||
75393 | 123 |
override def equals(that: Any): Boolean = { |
54440 | 124 |
that match { |
125 |
case other: Bytes => |
|
77712
dd4bb80dbc3a
tuned performance: much faster low-level operation;
wenzelm
parents:
77711
diff
changeset
|
126 |
this.eq(other) || |
dd4bb80dbc3a
tuned performance: much faster low-level operation;
wenzelm
parents:
77711
diff
changeset
|
127 |
Arrays.equals(bytes, offset, offset + length, |
dd4bb80dbc3a
tuned performance: much faster low-level operation;
wenzelm
parents:
77711
diff
changeset
|
128 |
other.bytes, other.offset, other.offset + other.length) |
54440 | 129 |
case _ => false |
130 |
} |
|
131 |
} |
|
132 |
||
75393 | 133 |
private lazy val hash: Int = { |
54439 | 134 |
var h = 0 |
135 |
for (i <- offset until offset + length) { |
|
136 |
val b = bytes(i).asInstanceOf[Int] & 0xFF |
|
137 |
h = 31 * h + b |
|
138 |
} |
|
139 |
h |
|
140 |
} |
|
141 |
||
142 |
override def hashCode(): Int = hash |
|
143 |
||
144 |
||
145 |
/* content */ |
|
146 |
||
54512 | 147 |
lazy val sha1_digest: SHA1.Digest = SHA1.digest(bytes) |
54440 | 148 |
|
69448 | 149 |
def is_empty: Boolean = length == 0 |
150 |
||
151 |
def iterator: Iterator[Byte] = |
|
152 |
for (i <- (offset until (offset + length)).iterator) |
|
153 |
yield bytes(i) |
|
154 |
||
75393 | 155 |
def array: Array[Byte] = { |
69365
c5b3860d29ef
avoid loading of font file, to eliminate "Illegal reflective access by com.lowagie.text.pdf.MappedRandomAccessFile$1 (iText-2.1.5.jar) to method java.nio.DirectByteBuffer.cleaner()" -- due to com.lowagie.text.pdf.TrueTypeFont.process() / RandomAccessFileOrArray;
wenzelm
parents:
68167
diff
changeset
|
156 |
val a = new Array[Byte](length) |
c5b3860d29ef
avoid loading of font file, to eliminate "Illegal reflective access by com.lowagie.text.pdf.MappedRandomAccessFile$1 (iText-2.1.5.jar) to method java.nio.DirectByteBuffer.cleaner()" -- due to com.lowagie.text.pdf.TrueTypeFont.process() / RandomAccessFileOrArray;
wenzelm
parents:
68167
diff
changeset
|
157 |
System.arraycopy(bytes, offset, a, 0, length) |
c5b3860d29ef
avoid loading of font file, to eliminate "Illegal reflective access by com.lowagie.text.pdf.MappedRandomAccessFile$1 (iText-2.1.5.jar) to method java.nio.DirectByteBuffer.cleaner()" -- due to com.lowagie.text.pdf.TrueTypeFont.process() / RandomAccessFileOrArray;
wenzelm
parents:
68167
diff
changeset
|
158 |
a |
c5b3860d29ef
avoid loading of font file, to eliminate "Illegal reflective access by com.lowagie.text.pdf.MappedRandomAccessFile$1 (iText-2.1.5.jar) to method java.nio.DirectByteBuffer.cleaner()" -- due to com.lowagie.text.pdf.TrueTypeFont.process() / RandomAccessFileOrArray;
wenzelm
parents:
68167
diff
changeset
|
159 |
} |
c5b3860d29ef
avoid loading of font file, to eliminate "Illegal reflective access by com.lowagie.text.pdf.MappedRandomAccessFile$1 (iText-2.1.5.jar) to method java.nio.DirectByteBuffer.cleaner()" -- due to com.lowagie.text.pdf.TrueTypeFont.process() / RandomAccessFileOrArray;
wenzelm
parents:
68167
diff
changeset
|
160 |
|
73561
c83152933579
clarified signature: Bytes extends CharSequence already (see d201996f72a8);
wenzelm
parents:
73559
diff
changeset
|
161 |
def text: String = UTF8.decode_permissive(this) |
65279
fa62e095d8f1
clarified signature (again, see also 3ed43cfc8b14);
wenzelm
parents:
65070
diff
changeset
|
162 |
|
76236 | 163 |
def wellformed_text: Option[String] = { |
164 |
val s = text |
|
165 |
if (this == Bytes(s)) Some(s) else None |
|
166 |
} |
|
167 |
||
75587 | 168 |
def encode_base64: String = { |
68094 | 169 |
val b = |
170 |
if (offset == 0 && length == bytes.length) bytes |
|
171 |
else Bytes(bytes, offset, length).bytes |
|
75620 | 172 |
Base64.encode(b) |
68094 | 173 |
} |
174 |
||
76236 | 175 |
def maybe_encode_base64: (Boolean, String) = |
176 |
wellformed_text match { |
|
177 |
case Some(s) => (false, s) |
|
178 |
case None => (true, encode_base64) |
|
179 |
} |
|
68106 | 180 |
|
77716 | 181 |
override def toString: String = |
182 |
if (is_empty) "Bytes.empty" else "Bytes(" + Space.bytes(length).print + ")" |
|
54439 | 183 |
|
72885 | 184 |
def proper: Option[Bytes] = if (is_empty) None else Some(this) |
185 |
def proper_text: Option[String] = if (is_empty) None else Some(text) |
|
65630 | 186 |
|
54439 | 187 |
def +(other: Bytes): Bytes = |
72885 | 188 |
if (other.is_empty) this |
189 |
else if (is_empty) other |
|
54439 | 190 |
else { |
191 |
val new_bytes = new Array[Byte](length + other.length) |
|
55618 | 192 |
System.arraycopy(bytes, offset, new_bytes, 0, length) |
193 |
System.arraycopy(other.bytes, other.offset, new_bytes, length, other.length) |
|
54439 | 194 |
new Bytes(new_bytes, 0, new_bytes.length) |
195 |
} |
|
54440 | 196 |
|
197 |
||
60833 | 198 |
/* CharSequence operations */ |
199 |
||
200 |
def charAt(i: Int): Char = |
|
201 |
if (0 <= i && i < length) (bytes(offset + i).asInstanceOf[Int] & 0xFF).asInstanceOf[Char] |
|
202 |
else throw new IndexOutOfBoundsException |
|
203 |
||
75393 | 204 |
def subSequence(i: Int, j: Int): Bytes = { |
60833 | 205 |
if (0 <= i && i <= j && j <= length) new Bytes(bytes, offset + i, j - i) |
206 |
else throw new IndexOutOfBoundsException |
|
207 |
} |
|
208 |
||
69448 | 209 |
def trim_line: Bytes = |
77714 | 210 |
if (length >= 2 && charAt(length - 2) == 13 && charAt(length - 1) == 10) { |
69448 | 211 |
subSequence(0, length - 2) |
77714 | 212 |
} |
213 |
else if (length >= 1 && (charAt(length - 1) == 13 || charAt(length - 1) == 10)) { |
|
69448 | 214 |
subSequence(0, length - 1) |
77714 | 215 |
} |
69448 | 216 |
else this |
217 |
||
60833 | 218 |
|
64004 | 219 |
/* streams */ |
220 |
||
221 |
def stream(): ByteArrayInputStream = new ByteArrayInputStream(bytes, offset, length) |
|
222 |
||
223 |
def write_stream(stream: OutputStream): Unit = stream.write(bytes, offset, length) |
|
224 |
||
225 |
||
76351
2cee31cd92f0
generic support for XZ and Zstd compression in Isabelle/Scala;
wenzelm
parents:
76350
diff
changeset
|
226 |
/* XZ / Zstd data compression */ |
2cee31cd92f0
generic support for XZ and Zstd compression in Isabelle/Scala;
wenzelm
parents:
76350
diff
changeset
|
227 |
|
76358 | 228 |
def detect_xz: Boolean = |
76351
2cee31cd92f0
generic support for XZ and Zstd compression in Isabelle/Scala;
wenzelm
parents:
76350
diff
changeset
|
229 |
length >= 6 && |
2cee31cd92f0
generic support for XZ and Zstd compression in Isabelle/Scala;
wenzelm
parents:
76350
diff
changeset
|
230 |
bytes(offset) == 0xFD.toByte && |
2cee31cd92f0
generic support for XZ and Zstd compression in Isabelle/Scala;
wenzelm
parents:
76350
diff
changeset
|
231 |
bytes(offset + 1) == 0x37.toByte && |
2cee31cd92f0
generic support for XZ and Zstd compression in Isabelle/Scala;
wenzelm
parents:
76350
diff
changeset
|
232 |
bytes(offset + 2) == 0x7A.toByte && |
2cee31cd92f0
generic support for XZ and Zstd compression in Isabelle/Scala;
wenzelm
parents:
76350
diff
changeset
|
233 |
bytes(offset + 3) == 0x58.toByte && |
2cee31cd92f0
generic support for XZ and Zstd compression in Isabelle/Scala;
wenzelm
parents:
76350
diff
changeset
|
234 |
bytes(offset + 4) == 0x5A.toByte && |
2cee31cd92f0
generic support for XZ and Zstd compression in Isabelle/Scala;
wenzelm
parents:
76350
diff
changeset
|
235 |
bytes(offset + 5) == 0x00.toByte |
2cee31cd92f0
generic support for XZ and Zstd compression in Isabelle/Scala;
wenzelm
parents:
76350
diff
changeset
|
236 |
|
76358 | 237 |
def detect_zstd: Boolean = |
76351
2cee31cd92f0
generic support for XZ and Zstd compression in Isabelle/Scala;
wenzelm
parents:
76350
diff
changeset
|
238 |
length >= 4 && |
2cee31cd92f0
generic support for XZ and Zstd compression in Isabelle/Scala;
wenzelm
parents:
76350
diff
changeset
|
239 |
bytes(offset) == 0x28.toByte && |
2cee31cd92f0
generic support for XZ and Zstd compression in Isabelle/Scala;
wenzelm
parents:
76350
diff
changeset
|
240 |
bytes(offset + 1) == 0xB5.toByte && |
2cee31cd92f0
generic support for XZ and Zstd compression in Isabelle/Scala;
wenzelm
parents:
76350
diff
changeset
|
241 |
bytes(offset + 2) == 0x2F.toByte && |
2cee31cd92f0
generic support for XZ and Zstd compression in Isabelle/Scala;
wenzelm
parents:
76350
diff
changeset
|
242 |
bytes(offset + 3) == 0xFD.toByte |
2cee31cd92f0
generic support for XZ and Zstd compression in Isabelle/Scala;
wenzelm
parents:
76350
diff
changeset
|
243 |
|
76358 | 244 |
def uncompress_xz(cache: Compress.Cache = Compress.Cache.none): Bytes = |
245 |
using(new xz.XZInputStream(stream(), cache.for_xz))(Bytes.read_stream(_, hint = length)) |
|
246 |
||
247 |
def uncompress_zstd(cache: Compress.Cache = Compress.Cache.none): Bytes = { |
|
248 |
Zstd.init() |
|
249 |
val n = zstd.Zstd.decompressedSize(bytes, offset, length) |
|
78243 | 250 |
if (n > 0 && n < Int.MaxValue) { |
76358 | 251 |
Bytes(zstd.Zstd.decompress(array, n.toInt)) |
252 |
} |
|
253 |
else { |
|
254 |
using(new zstd.ZstdInputStream(stream(), cache.for_zstd))(Bytes.read_stream(_, hint = length)) |
|
255 |
} |
|
256 |
} |
|
54440 | 257 |
|
76351
2cee31cd92f0
generic support for XZ and Zstd compression in Isabelle/Scala;
wenzelm
parents:
76350
diff
changeset
|
258 |
def uncompress(cache: Compress.Cache = Compress.Cache.none): Bytes = |
76358 | 259 |
if (detect_xz) uncompress_xz(cache = cache) |
260 |
else if (detect_zstd) uncompress_zstd(cache = cache) |
|
261 |
else error("Cannot detect compression scheme") |
|
64004 | 262 |
|
76351
2cee31cd92f0
generic support for XZ and Zstd compression in Isabelle/Scala;
wenzelm
parents:
76350
diff
changeset
|
263 |
def compress( |
2cee31cd92f0
generic support for XZ and Zstd compression in Isabelle/Scala;
wenzelm
parents:
76350
diff
changeset
|
264 |
options: Compress.Options = Compress.Options(), |
2cee31cd92f0
generic support for XZ and Zstd compression in Isabelle/Scala;
wenzelm
parents:
76350
diff
changeset
|
265 |
cache: Compress.Cache = Compress.Cache.none |
2cee31cd92f0
generic support for XZ and Zstd compression in Isabelle/Scala;
wenzelm
parents:
76350
diff
changeset
|
266 |
): Bytes = { |
76358 | 267 |
options match { |
268 |
case options_xz: Compress.Options_XZ => |
|
269 |
val result = new ByteArrayOutputStream(length) |
|
270 |
using(new xz.XZOutputStream(result, options_xz.make, cache.for_xz))(write_stream) |
|
271 |
new Bytes(result.toByteArray, 0, result.size) |
|
272 |
case options_zstd: Compress.Options_Zstd => |
|
273 |
Zstd.init() |
|
76361
3b9f36ef7365
tuned: avoid redundant copy of potentially large array;
wenzelm
parents:
76358
diff
changeset
|
274 |
Bytes(zstd.Zstd.compress(if (offset == 0) bytes else array, options_zstd.level)) |
76358 | 275 |
} |
64004 | 276 |
} |
68167 | 277 |
|
75393 | 278 |
def maybe_compress( |
76351
2cee31cd92f0
generic support for XZ and Zstd compression in Isabelle/Scala;
wenzelm
parents:
76350
diff
changeset
|
279 |
options: Compress.Options = Compress.Options(), |
2cee31cd92f0
generic support for XZ and Zstd compression in Isabelle/Scala;
wenzelm
parents:
76350
diff
changeset
|
280 |
cache: Compress.Cache = Compress.Cache.none |
75393 | 281 |
) : (Boolean, Bytes) = { |
68167 | 282 |
val compressed = compress(options = options, cache = cache) |
283 |
if (compressed.length < length) (true, compressed) else (false, this) |
|
284 |
} |
|
54439 | 285 |
} |