2 * Copyright (C) 2004 Bas Peters
\r
4 * This file is part of MARC4J
\r
6 * MARC4J is free software; you can redistribute it and/or
\r
7 * modify it under the terms of the GNU Lesser General Public
\r
8 * License as published by the Free Software Foundation; either
\r
9 * version 2.1 of the License, or (at your option) any later version.
\r
11 * MARC4J is distributed in the hope that it will be useful,
\r
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
\r
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
\r
14 * Lesser General Public License for more details.
\r
16 * You should have received a copy of the GNU Lesser General Public
\r
17 * License along with MARC4J; if not, write to the Free Software
\r
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
\r
22 import java.io.BufferedInputStream;
\r
23 import java.io.ByteArrayInputStream;
\r
24 import java.io.DataInputStream;
\r
25 import java.io.EOFException;
\r
26 import java.io.IOException;
\r
27 import java.io.InputStream;
\r
28 import java.io.InputStreamReader;
\r
29 import java.io.UnsupportedEncodingException;
\r
30 import java.util.Iterator;
\r
31 import java.util.List;
\r
32 import java.util.regex.Matcher;
\r
33 import java.util.regex.Pattern;
\r
35 import org.marc4j.converter.CharConverter;
\r
36 import org.marc4j.converter.impl.AnselToUnicode;
\r
37 import org.marc4j.converter.impl.Iso5426ToUnicode;
\r
38 import org.marc4j.marc.ControlField;
\r
39 import org.marc4j.marc.DataField;
\r
40 import org.marc4j.marc.Leader;
\r
41 import org.marc4j.marc.MarcFactory;
\r
42 import org.marc4j.marc.Record;
\r
43 import org.marc4j.marc.Subfield;
\r
44 import org.marc4j.marc.VariableField;
\r
45 import org.marc4j.marc.impl.Verifier;
\r
47 import com.ibm.icu.text.Normalizer;
\r
50 * An iterator over a collection of MARC records in ISO 2709 format, that is designed
\r
51 * to be able to handle MARC records that have errors in their structure or their encoding.
\r
52 * If the permissive flag is set in the call to the constructor, or if a ErrorHandler object
\r
53 * is passed in as a parameter to the constructor, this reader will do its best to detect
\r
54 * and recover from a number of structural or encoding errors that can occur in a MARC record.
\r
55 * Note that if this reader is not set to read permissively, its will operate pretty much
\r
56 * identically to the MarcStreamReader class.
\r
58 * Note that no attempt is made to validate the contents of the record at a semantic level.
\r
59 * This reader does not know and does not care whether the record has a 245 field, or if the
\r
60 * 008 field is the right length, but if the record claims to be UTF-8 or MARC8 encoded and
\r
61 * you are seeing gibberish in the output, or if the reader is throwing an exception in trying
\r
62 * to read a record, then this reader may be able to produce a usable record from the bad
\r
65 * The ability to directly translate the record to UTF-8 as it is being read in is useful in
\r
66 * cases where the UTF-8 version of the record will be used directly by the program that is
\r
67 * reading the MARC data, for instance if the marc records are to be indexed into a SOLR search
\r
68 * engine. Previously the MARC record could only be translated to UTF-8 as it was being written
\r
69 * out via a MarcStreamWriter or a MarcXmlWriter.
\r
75 * InputStream input = new FileInputStream("file.mrc");
\r
76 * MarcReader reader = new MarcPermissiveStreamReader(input, true, true);
\r
77 * while (reader.hasNext()) {
\r
78 * Record record = reader.next();
\r
84 * Check the {@link org.marc4j.marc} package for examples about the use of
\r
85 * the {@link org.marc4j.marc.Record} object model.
\r
86 * Check the file org.marc4j.samples.PermissiveReaderExample.java for an
\r
87 * example about using the MarcPermissiveStreamReader in conjunction with the
\r
88 * ErrorHandler class to report errors encountered while processing records.
\r
92 * When no encoding is given as an constructor argument the parser tries to
\r
93 * resolve the encoding by looking at the character coding scheme (leader
\r
94 * position 9) in MARC21 records. For UNIMARC records this position is not
\r
95 * defined. If the reader is operating in permissive mode and no encoding
\r
96 * is given as an constructor argument the reader will look at the leader,
\r
97 * and also at the data of the record to determine to the best of its ability
\r
98 * what character encoding scheme has been used to encode the data in a
\r
99 * particular MARC record.
\r
103 * @author Robert Haschart
\r
104 * @version $Revision: 1.3 $
\r
107 public class MarcPermissiveStreamReader implements MarcReader {
\r
109 private DataInputStream input = null;
\r
111 private Record record;
\r
113 private MarcFactory factory;
\r
115 private String encoding = "ISO8859_1";
\r
117 // This represents the expected encoding of the data when a
\r
118 // MARC record does not have a 'a' in character 9 of the leader.
\r
119 private String defaultEncoding = "ISO8859_1";
\r
121 private boolean convertToUTF8 = false;
\r
123 private boolean permissive = false;
\r
125 private CharConverter converterAnsel = null;
\r
127 private CharConverter converterUnimarc = null;
\r
129 // These are used to algorithmically determine what encoding scheme was
\r
130 // used to encode the data in the Marc record
\r
131 private String conversionCheck1 = null;
\r
132 private String conversionCheck2 = null;
\r
133 private String conversionCheck3 = null;
\r
135 private ErrorHandler errors;
\r
138 * Constructs an instance with the specified input stream with possible additional functionality
\r
139 * being enabled by setting permissive and/or convertToUTF8 to true.
\r
141 * If permissive and convertToUTF8 are both set to false, it functions almost identically to the
\r
142 * MarcStreamReader class.
\r
144 public MarcPermissiveStreamReader(InputStream input, boolean permissive, boolean convertToUTF8) {
\r
145 this.permissive = permissive;
\r
146 this.input = new DataInputStream(new BufferedInputStream(input));
\r
147 factory = MarcFactory.newInstance();
\r
148 this.convertToUTF8 = convertToUTF8;
\r
152 errors = new ErrorHandler();
\r
153 defaultEncoding = "BESTGUESS";
\r
158 * Constructs an instance with the specified input stream with possible additional functionality
\r
159 * being enabled by passing in an ErrorHandler object and/or setting convertToUTF8 to true.
\r
161 * If errors and convertToUTF8 are both set to false, it functions almost identically to the
\r
162 * MarcStreamReader class.
\r
164 * If an ErrorHandler object is passed in, that object will be used to log and track any errors
\r
165 * in the records as the records are decoded. After the next() function returns, you can query
\r
166 * to determine whether any errors were detected in the decoding process.
\r
168 * See the file org.marc4j.samples.PermissiveReaderExample.java to see how this can be done.
\r
170 public MarcPermissiveStreamReader(InputStream input, ErrorHandler errors, boolean convertToUTF8 )
\r
172 if (errors != null)
\r
175 defaultEncoding = "BESTGUESS";
\r
177 this.input = new DataInputStream(new BufferedInputStream(input));
\r
178 factory = MarcFactory.newInstance();
\r
179 this.convertToUTF8 = convertToUTF8;
\r
180 this.errors = errors;
\r
184 * Constructs an instance with the specified input stream with possible additional functionality
\r
185 * being enabled by setting permissive and/or convertToUTF8 to true.
\r
187 * If permissive and convertToUTF8 are both set to false, it functions almost identically to the
\r
188 * MarcStreamReader class.
\r
190 * The parameter defaultEncoding is used to specify the character encoding that is used in the records
\r
191 * that will be read from the input stream. If permissive is set to true, you can specify "BESTGUESS"
\r
192 * as the default encoding, and the reader will attempt to determine the character encoding used in the
\r
193 * records being read from the input stream. This is especially useful if you are working with records
\r
194 * downloaded from an external source and the encoding is either unknown or the encoding is different from
\r
195 * what the records claim to be.
\r
197 public MarcPermissiveStreamReader(InputStream input, boolean permissive, boolean convertToUTF8, String defaultEncoding)
\r
199 this.permissive = permissive;
\r
200 this.input = new DataInputStream(new BufferedInputStream(input));
\r
201 factory = MarcFactory.newInstance();
\r
202 this.convertToUTF8 = convertToUTF8;
\r
203 this.defaultEncoding = defaultEncoding;
\r
205 if (permissive) errors = new ErrorHandler();
\r
209 * Constructs an instance with the specified input stream with possible additional functionality
\r
210 * being enabled by setting permissive and/or convertToUTF8 to true.
\r
212 * If errors and convertToUTF8 are both set to false, it functions almost identically to the
\r
213 * MarcStreamReader class.
\r
215 * The parameter defaultEncoding is used to specify the character encoding that is used in the records
\r
216 * that will be read from the input stream. If permissive is set to true, you can specify "BESTGUESS"
\r
217 * as the default encoding, and the reader will attempt to determine the character encoding used in the
\r
218 * records being read from the input stream. This is especially useful if you are working with records
\r
219 * downloaded from an external source and the encoding is either unknown or the encoding is different from
\r
220 * what the records claim to be.
\r
222 * If an ErrorHandler object is passed in, that object will be used to log and track any errors
\r
223 * in the records as the records are decoded. After the next() function returns, you can query
\r
224 * to determine whether any errors were detected in the decoding process.
\r
226 * See the file org.marc4j.samples.PermissiveReaderExample.java to see how this can be done.
\r
228 public MarcPermissiveStreamReader(InputStream input, ErrorHandler errors, boolean convertToUTF8, String defaultEncoding)
\r
230 this.permissive = true;
\r
231 this.input = new DataInputStream(new BufferedInputStream(input));
\r
232 factory = MarcFactory.newInstance();
\r
233 this.convertToUTF8 = convertToUTF8;
\r
234 this.defaultEncoding = defaultEncoding;
\r
235 this.errors = errors;
\r
239 * Returns true if the iteration has more records, false otherwise.
\r
241 public boolean hasNext()
\r
244 if (input.available() == 0)
\r
246 } catch (IOException e) {
\r
247 throw new MarcException(e.getMessage(), e);
\r
253 * Returns the next record in the iteration.
\r
255 * @return Record - the record object
\r
257 public Record next()
\r
259 record = factory.newRecord();
\r
260 if (errors != null) errors.reset();
\r
263 byte[] byteArray = new byte[24];
\r
264 input.readFully(byteArray);
\r
266 int recordLength = parseRecordLength(byteArray);
\r
267 byte[] recordBuf = new byte[recordLength - 24];
\r
270 input.mark(recordLength * 2);
\r
271 input.readFully(recordBuf);
\r
272 if (recordBuf[recordBuf.length-1] != Constants.RT)
\r
274 errors.addError("unknown", "n/a", "n/a", ErrorHandler.MAJOR_ERROR,
\r
275 "Record terminator character not found at end of record length");
\r
276 recordBuf = rereadPermissively(input, recordBuf, recordLength);
\r
277 recordLength = recordBuf.length + 24;
\r
282 input.readFully(recordBuf);
\r
284 //String tmp = new String(recordBuf);
\r
285 parseRecord(record, byteArray, recordBuf, recordLength);
\r
287 if (this.convertToUTF8)
\r
289 Leader l = record.getLeader();
\r
290 l.setCharCodingScheme('a');
\r
291 record.setLeader(l);
\r
295 catch (EOFException e) {
\r
296 throw new MarcException("Premature end of file encountered", e);
\r
298 catch (IOException e) {
\r
299 throw new MarcException("an error occured reading input", e);
\r
303 private byte[] rereadPermissively(DataInputStream input, byte[] recordBuf, int recordLength) throws IOException
\r
305 int loc = arrayContainsAt(recordBuf, Constants.RT);
\r
306 if (loc != -1) // stated record length is too long
\r
308 errors.addError("unknown", "n/a", "n/a", ErrorHandler.MAJOR_ERROR,
\r
309 "Record terminator appears before stated record length, using shorter record");
\r
310 recordLength = loc + 24;
\r
312 recordBuf = new byte[recordLength - 24];
\r
313 input.readFully(recordBuf);
\r
315 else // stated record length is too short read ahead
\r
317 loc = recordLength - 24;
\r
323 } while (loc < recordLength + 100 && c != Constants.RT && c != -1);
\r
325 if (c == Constants.RT)
\r
327 errors.addError("unknown", "n/a", "n/a", ErrorHandler.MAJOR_ERROR,
\r
328 "Record terminator appears after stated record length, reading extra bytes");
\r
329 recordLength = loc + 24;
\r
331 recordBuf = new byte[recordLength - 24];
\r
332 input.readFully(recordBuf);
\r
336 errors.addError("unknown", "n/a", "n/a", ErrorHandler.MAJOR_ERROR,
\r
337 "No Record terminator found, end of file reached, Terminator appended");
\r
338 recordLength = loc + 24;
\r
340 recordBuf = new byte[recordLength - 24 + 1];
\r
341 input.readFully(recordBuf);
\r
342 recordBuf[recordBuf.length-1] = Constants.RT;
\r
346 errors.addError("unknown", "n/a", "n/a", ErrorHandler.FATAL,
\r
347 "No Record terminator found within 100 byts of stated location, giving up.");
\r
353 private void parseRecord(Record record, byte[] byteArray, byte[] recordBuf, int recordLength)
\r
356 ldr = factory.newLeader();
\r
357 ldr.setRecordLength(recordLength);
\r
358 int directoryLength=0;
\r
359 // These variables are used when the permissive reader is trying to make its best guess
\r
360 // as to what character encoding is actually used in the record being processed.
\r
361 conversionCheck1 = "";
\r
362 conversionCheck2 = "";
\r
363 conversionCheck3 = "";
\r
366 parseLeader(ldr, byteArray);
\r
367 directoryLength = ldr.getBaseAddressOfData() - (24 + 1);
\r
369 catch (IOException e) {
\r
370 throw new MarcException("error parsing leader with data: "
\r
371 + new String(byteArray), e);
\r
373 catch (MarcException e) {
\r
376 if (recordBuf[recordBuf.length-1] == Constants.RT && recordBuf[recordBuf.length-2] == Constants.FT)
\r
378 errors.addError("unknown", "n/a", "n/a", ErrorHandler.MAJOR_ERROR,
\r
379 "Error parsing leader, trying to re-read leader either shorter or longer");
\r
380 // make an attempt to recover record.
\r
382 while (offset < recordBuf.length)
\r
384 if (recordBuf[offset] == Constants.FT)
\r
390 if (offset % 12 == 1)
\r
392 // move one byte from body to leader, make new leader, and try again
\r
393 errors.addError("unknown", "n/a", "n/a", ErrorHandler.MAJOR_ERROR,
\r
394 "Leader appears to be too short, moving one byte from record body to leader, and trying again");
\r
395 byte oldBody[] = recordBuf;
\r
396 recordBuf = new byte[oldBody.length-1];
\r
397 System.arraycopy(oldBody, 1, recordBuf, 0, oldBody.length-1);
\r
398 directoryLength = offset-1;
\r
399 ldr.setIndicatorCount(2);
\r
400 ldr.setSubfieldCodeLength(2);
\r
401 ldr.setImplDefined1((""+(char)byteArray[7]+" ").toCharArray());
\r
402 ldr.setImplDefined2((""+(char)byteArray[18]+(char)byteArray[19]+(char)byteArray[20]).toCharArray());
\r
403 ldr.setEntryMap("4500".toCharArray());
\r
404 if (byteArray[10] == (byte)' ' || byteArray[10] == (byte)'a') // if its ' ' or 'a'
\r
406 ldr.setCharCodingScheme((char)byteArray[10]);
\r
409 else if (offset % 12 == 11)
\r
411 errors.addError("unknown", "n/a", "n/a", ErrorHandler.MAJOR_ERROR,
\r
412 "Leader appears to be too long, moving one byte from leader to record body, and trying again");
\r
413 byte oldBody[] = recordBuf;
\r
414 recordBuf = new byte[oldBody.length+1];
\r
415 System.arraycopy(oldBody, 0, recordBuf, 1, oldBody.length);
\r
416 recordBuf[0] = (byte)'0';
\r
417 directoryLength = offset+1;
\r
418 ldr.setIndicatorCount(2);
\r
419 ldr.setSubfieldCodeLength(2);
\r
420 ldr.setImplDefined1((""+(char)byteArray[7]+" ").toCharArray());
\r
421 ldr.setImplDefined2((""+(char)byteArray[16]+(char)byteArray[17]+(char)byteArray[18]).toCharArray());
\r
422 ldr.setEntryMap("4500".toCharArray());
\r
423 if (byteArray[8] == (byte)' ' || byteArray[8] == (byte)'a') // if its ' ' or 'a'
\r
425 ldr.setCharCodingScheme((char)byteArray[10]);
\r
427 if (byteArray[10] == (byte)' ' || byteArray[10] == (byte)'a') // if its ' ' or 'a'
\r
429 ldr.setCharCodingScheme((char)byteArray[10]);
\r
434 errors.addError("unknown", "n/a", "n/a", ErrorHandler.FATAL,
\r
435 "error parsing leader with data: " + new String(byteArray));
\r
436 throw new MarcException("error parsing leader with data: "
\r
437 + new String(byteArray), e);
\r
443 throw new MarcException("error parsing leader with data: "
\r
444 + new String(byteArray), e);
\r
447 char tmp[] = ldr.getEntryMap();
\r
448 if (permissive && !(""+ tmp[0]+tmp[1]+tmp[2]+tmp[3]).equals("4500"))
\r
450 if (tmp[0] >= '0' && tmp[0] <= '9' &&
\r
451 tmp[1] >= '0' && tmp[1] <= '9' &&
\r
452 tmp[2] >= '0' && tmp[2] <= '9' &&
\r
453 tmp[3] >= '0' && tmp[3] <= '9')
\r
455 errors.addError("unknown", "n/a", "n/a", ErrorHandler.ERROR_TYPO,
\r
456 "Unusual character found at end of leader [ "+tmp[0]+tmp[1]+tmp[2]+tmp[3]+" ]");
\r
460 errors.addError("unknown", "n/a", "n/a", ErrorHandler.ERROR_TYPO,
\r
461 "Erroneous character found at end of leader [ "+tmp[0]+tmp[1]+tmp[2]+tmp[3]+" ]; changing them to the standard \"4500\"");
\r
462 ldr.setEntryMap("4500".toCharArray());
\r
466 // if MARC 21 then check encoding
\r
467 switch (ldr.getCharCodingScheme()) {
\r
473 encoding = defaultEncoding;
\r
475 encoding = "ISO8859_1";
\r
479 encoding = defaultEncoding;
\r
481 encoding = "ISO8859_1";
\r
486 if (encoding.equalsIgnoreCase("BESTGUESS"))
\r
490 String marc8EscSeqCheck = new String(recordBuf, "ISO-8859-1");
\r
491 // If record has MARC8 character set selection strings, it must be MARC8 encoded
\r
492 if (marc8EscSeqCheck.split("\\e[-(,)$bsp]", 2).length > 1)
\r
494 encoding = "MARC8";
\r
498 boolean hasHighBitChars = false;
\r
499 for (int i = 0; i < recordBuf.length; i++)
\r
501 if (recordBuf[i] < 0) // the high bit is set
\r
503 hasHighBitChars = true;
\r
507 if (!hasHighBitChars)
\r
509 encoding = "ISO8859_1"; // You can choose any encoding you want here, the results will be the same.
\r
513 utfCheck = new String(recordBuf, "UTF-8");
\r
514 byte byteCheck[] = utfCheck.getBytes("UTF-8");
\r
515 encoding = "UTF8";
\r
516 if (recordBuf.length == byteCheck.length)
\r
518 for (int i = 0; i < recordBuf.length; i++)
\r
520 if (byteCheck[i] != recordBuf[i])
\r
522 encoding = "MARC8-Maybe";
\r
529 encoding = "MARC8-Maybe";
\r
534 catch (UnsupportedEncodingException e)
\r
536 // TODO Auto-generated catch block
\r
537 e.printStackTrace();
\r
540 else if (permissive && encoding.equals("UTF8"))
\r
544 utfCheck = new String(recordBuf, "UTF-8");
\r
545 byte byteCheck[] = utfCheck.getBytes("UTF-8");
\r
546 if (recordBuf.length != byteCheck.length)
\r
548 boolean foundESC = false;
\r
549 for (int i = 0; i < recordBuf.length; i++)
\r
551 if (recordBuf[i] == 0x1B)
\r
553 errors.addError("unknown", "n/a", "n/a", ErrorHandler.MINOR_ERROR,
\r
554 "Record claims to be UTF-8, but its not. Its probably MARC8.");
\r
555 encoding = "MARC8-Maybe";
\r
559 if (byteCheck[i] != recordBuf[i])
\r
561 encoding = "MARC8-Maybe";
\r
567 errors.addError("unknown", "n/a", "n/a", ErrorHandler.MINOR_ERROR,
\r
568 "Record claims to be UTF-8, but its not. It may be MARC8, or maybe UNIMARC, or maybe raw ISO-8859-1 ");
\r
571 if (utfCheck.contains("a$1!"))
\r
573 encoding = "MARC8-Broken";
\r
574 errors.addError("unknown", "n/a", "n/a", ErrorHandler.MAJOR_ERROR,
\r
575 "Record claims to be UTF-8, but its not. It seems to be MARC8-encoded but with missing escape codes.");
\r
578 catch (UnsupportedEncodingException e)
\r
580 // TODO Auto-generated catch block
\r
581 e.printStackTrace();
\r
584 else if (permissive && !encoding.equals("UTF8"))
\r
588 utfCheck = new String(recordBuf, "UTF-8");
\r
589 byte byteCheck[] = utfCheck.getBytes("UTF-8");
\r
590 if (recordBuf.length == byteCheck.length)
\r
592 for (int i = 0; i < recordBuf.length; i++)
\r
594 // need to check for byte < 0 to see if the high bit is set, because Java doesn't have unsigned types.
\r
595 if (recordBuf[i] < 0x00 || byteCheck[i] != recordBuf[i])
\r
597 errors.addError("unknown", "n/a", "n/a", ErrorHandler.MINOR_ERROR,
\r
598 "Record claims not to be UTF-8, but it seems to be.");
\r
599 encoding = "UTF8-Maybe";
\r
605 catch (UnsupportedEncodingException e)
\r
607 // TODO Auto-generated catch block
\r
608 e.printStackTrace();
\r
611 record.setLeader(ldr);
\r
613 boolean discardOneAtStartOfDirectory = false;
\r
614 boolean discardOneSomewhereInDirectory = false;
\r
616 if ((directoryLength % 12) != 0)
\r
618 if (permissive && directoryLength % 12 == 11 && recordBuf[1] != (byte)'0')
\r
620 errors.addError("unknown", "n/a", "n/a", ErrorHandler.MAJOR_ERROR,
\r
621 "Directory length is not a multiple of 12 bytes long. Prepending a zero and trying to continue.");
\r
622 byte oldBody[] = recordBuf;
\r
623 recordBuf = new byte[oldBody.length+1];
\r
624 System.arraycopy(oldBody, 0, recordBuf, 1, oldBody.length);
\r
625 recordBuf[0] = (byte)'0';
\r
626 directoryLength = directoryLength+1;
\r
630 if (permissive && directoryLength % 12 == 1 && recordBuf[1] == (byte)'0' && recordBuf[2] == (byte)'0')
\r
632 discardOneAtStartOfDirectory = true;
\r
633 errors.addError("unknown", "n/a", "n/a", ErrorHandler.MAJOR_ERROR,
\r
634 "Directory length is not a multiple of 12 bytes long. Discarding byte from start of directory and trying to continue.");
\r
636 else if (permissive && directoryLength % 12 == 1 && recordLength > 10000 && recordBuf[0] == (byte)'0' &&
\r
637 recordBuf[1] == (byte)'0' && recordBuf[2] > (byte)'0' && recordBuf[2] <= (byte)'9')
\r
639 discardOneSomewhereInDirectory = true;
\r
640 errors.addError("unknown", "n/a", "n/a", ErrorHandler.MAJOR_ERROR,
\r
641 "Directory length is not a multiple of 12 bytes long. Will look for oversized field and try to work around it.");
\r
645 if (errors != null)
\r
647 errors.addError("unknown", "n/a", "n/a", ErrorHandler.FATAL,
\r
648 "Directory length is not a multiple of 12 bytes long. Unable to continue.");
\r
650 throw new MarcException("Directory length is not a multiple of 12 bytes long. Unable to continue.");
\r
654 DataInputStream inputrec = new DataInputStream(new ByteArrayInputStream(recordBuf));
\r
655 int size = directoryLength / 12;
\r
657 String[] tags = new String[size];
\r
658 int[] lengths = new int[size];
\r
660 byte[] tag = new byte[3];
\r
661 byte[] length = new byte[4];
\r
662 byte[] start = new byte[5];
\r
666 if (discardOneAtStartOfDirectory) inputrec.read();
\r
667 int totalOffset = 0;
\r
668 for (int i = 0; i < size; i++)
\r
670 inputrec.readFully(tag);
\r
671 tmpStr = new String(tag);
\r
674 boolean proceedNormally = true;
\r
675 if (discardOneSomewhereInDirectory)
\r
677 byte lenCheck[] = new byte[10];
\r
679 inputrec.readFully(lenCheck);
\r
680 if (byteCompare(lenCheck, 4, 5, totalOffset)) // proceed normally
\r
682 proceedNormally = true;
\r
684 else if (byteCompare(lenCheck, 5, 5, totalOffset)) // field length is 5 bytes! Bad Marc record, proceed normally
\r
686 discardOneSomewhereInDirectory = false;
\r
687 errors.addError("unknown", "n/a", "n/a", ErrorHandler.FATAL,
\r
688 "Field is longer than 9999 bytes. Writing this record out will result in a bad record.");
\r
689 proceedNormally = false;
\r
693 errors.addError("unknown", "n/a", "n/a", ErrorHandler.FATAL,
\r
694 "Unable to reconcile problems in directory. Unable to continue.");
\r
695 throw new MarcException("Directory length is not a multiple of 12 bytes long. Unable to continue.");
\r
699 if (proceedNormally)
\r
701 inputrec.readFully(length);
\r
702 tmpStr = new String(length);
\r
703 lengths[i] = Integer.parseInt(tmpStr);
\r
705 inputrec.readFully(start);
\r
707 else // length is 5 bytes long
\r
709 inputrec.readFully(start);
\r
710 tmpStr = new String(start);
\r
711 lengths[i] = Integer.parseInt(tmpStr);
\r
713 inputrec.readFully(start);
\r
715 totalOffset += lengths[i];
\r
718 // If we still haven't found the extra byte, throw out the last byte and try to continue;
\r
719 if (discardOneSomewhereInDirectory) inputrec.read();
\r
721 if (inputrec.read() != Constants.FT)
\r
723 errors.addError("unknown", "n/a", "n/a", ErrorHandler.FATAL,
\r
724 "Expected field terminator at end of directory. Unable to continue.");
\r
725 throw new MarcException("expected field terminator at end of directory");
\r
728 int numBadLengths = 0;
\r
730 int totalLength = 0;
\r
731 for (int i = 0; i < size; i++)
\r
733 int fieldLength = getFieldLength(inputrec);
\r
734 if (fieldLength+1 != lengths[i] && permissive)
\r
736 if (numBadLengths < 3 && (totalLength + fieldLength < recordLength + 26))
\r
739 lengths[i] = fieldLength+1;
\r
740 errors.addError("unknown", "n/a", "n/a", ErrorHandler.MINOR_ERROR,
\r
741 "Field length found in record different from length stated in the directory.");
\r
742 if (fieldLength+1 > 9999)
\r
744 errors.addError("unknown", "n/a", "n/a", ErrorHandler.FATAL,
\r
745 "Field length is greater than 9999, record cannot be represented as a binary Marc record.");
\r
749 totalLength += lengths[i];
\r
750 if (isControlField(tags[i]))
\r
752 byteArray = new byte[lengths[i] - 1];
\r
753 inputrec.readFully(byteArray);
\r
755 if (inputrec.read() != Constants.FT)
\r
757 errors.addError("unknown", "n/a", "n/a", ErrorHandler.FATAL,
\r
758 "Expected field terminator at end of field. Unable to continue.");
\r
759 throw new MarcException("expected field terminator at end of field");
\r
762 ControlField field = factory.newControlField();
\r
763 field.setTag(tags[i]);
\r
764 field.setData(getDataAsString(byteArray));
\r
765 record.addVariableField(field);
\r
770 byteArray = new byte[lengths[i]];
\r
771 inputrec.readFully(byteArray);
\r
773 record.addVariableField(parseDataField(tags[i], byteArray));
\r
774 } catch (IOException e) {
\r
775 throw new MarcException(
\r
776 "error parsing data field for tag: " + tags[i]
\r
778 + new String(byteArray), e);
\r
783 // We've determined that although the record says it is UTF-8, it is not.
\r
784 // Here we make an attempt to determine the actual encoding of the data in the record.
\r
785 if (permissive && conversionCheck1.length() > 1 &&
\r
786 conversionCheck2.length() > 1 && conversionCheck3.length() > 1)
\r
788 guessAndSelectCorrectNonUTF8Encoding();
\r
790 if (inputrec.read() != Constants.RT)
\r
792 errors.addError("unknown", "n/a", "n/a", ErrorHandler.FATAL,
\r
793 "Expected record terminator at end of record. Unable to continue.");
\r
794 throw new MarcException("expected record terminator");
\r
797 catch (IOException e)
\r
799 errors.addError("unknown", "n/a", "n/a", ErrorHandler.FATAL,
\r
800 "Error reading from data file. Unable to continue.");
\r
801 throw new MarcException("an error occured reading input", e);
\r
805 private boolean byteCompare(byte[] lenCheck, int offset, int length, int totalOffset)
\r
808 for (int i = offset + length - 1; i >= offset; i-- , divisor *= 10)
\r
810 if (((totalOffset / divisor) % 10) + '0' != lenCheck[i])
\r
818 private boolean isControlField(String tag)
\r
820 boolean isControl = false;
\r
822 isControl = Verifier.isControlField(tag);
\r
824 catch (NumberFormatException nfe)
\r
828 errors.addError(record.getControlNumber(), tag, "n/a", ErrorHandler.ERROR_TYPO,
\r
829 "Field tag contains non-numeric characters (" + tag + ").");
\r
836 private void guessAndSelectCorrectNonUTF8Encoding()
\r
838 int defaultPart = 0;
\r
839 if (record.getVariableField("245") == null) defaultPart = 1;
\r
841 int l1 = conversionCheck1.length();
\r
842 int l2 = conversionCheck2.length();
\r
843 int l3 = conversionCheck3.length();
\r
846 if (l1 < l3 && l2 == l3 && defaultPart == 0)
\r
848 errors.addError(ErrorHandler.INFO, "MARC8 translation shorter than ISO-8859-1, choosing MARC8.");
\r
851 else if (l2 < l1-2 && l2 < l3-2 )
\r
853 errors.addError(ErrorHandler.INFO, "Unimarc translation shortest, choosing it.");
\r
856 else if ((tst = onlyOneStartsWithUpperCase(conversionCheck1, conversionCheck2, conversionCheck3)) != -1)
\r
860 else if (l2 < l1 && l2 < l3 )
\r
862 errors.addError(ErrorHandler.INFO, "Unimarc translation shortest, choosing it.");
\r
865 else if (conversionCheck2.equals(conversionCheck3) && !conversionCheck1.trim().contains(" "))
\r
867 errors.addError(ErrorHandler.INFO, "Unimarc and ISO-8859-1 translations identical, choosing ISO-8859-1.");
\r
870 else if (!specialCharIsBetweenLetters(conversionCheck1))
\r
872 errors.addError(ErrorHandler.INFO, "To few letters in translations, choosing "+(defaultPart == 0 ? "MARC8" : "Unimarc"));
\r
873 partToUse = defaultPart;
\r
875 else if (l2 == l1 && l2 == l3)
\r
877 errors.addError(ErrorHandler.INFO, "All three version equal length. Choosing ISO-8859-1 ");
\r
880 else if (l2 == l3 && defaultPart == 1)
\r
882 errors.addError(ErrorHandler.INFO, "Unimarc and ISO-8859-1 translations equal length, choosing ISO-8859-1.");
\r
887 errors.addError(ErrorHandler.INFO, "No Determination made, defaulting to "+ (defaultPart == 0 ? "MARC8" : "Unimarc") );
\r
888 partToUse = defaultPart;
\r
890 List<VariableField> fields = record.getVariableFields();
\r
891 Iterator<VariableField> iter = fields.iterator();
\r
892 while (iter.hasNext())
\r
894 VariableField field = iter.next();
\r
895 if (field instanceof DataField)
\r
897 DataField df = (DataField)field;
\r
898 List<Subfield> subf = df.getSubfields();
\r
899 Iterator<Subfield> sfiter = subf.iterator();
\r
900 while (sfiter.hasNext())
\r
902 Subfield sf = sfiter.next();
\r
903 if (sf.getData().contains("%%@%%"))
\r
905 String parts[] = sf.getData().split("%%@%%", 3);
\r
906 sf.setData(parts[partToUse]);
\r
913 private int onlyOneStartsWithUpperCase(String conversionCheck12, String conversionCheck22, String conversionCheck32)
\r
915 if (conversionCheck1.length() == 0 || conversionCheck2.length() == 0 || conversionCheck3.length() == 0) return -1;
\r
916 String check1Parts[] = conversionCheck1.trim().split("[|]>");
\r
917 String check2Parts[] = conversionCheck2.trim().split("[|]>");
\r
918 String check3Parts[] = conversionCheck3.trim().split("[|]>");
\r
919 for (int i = 1; i < check1Parts.length && i < check2Parts.length && i < check3Parts.length; i++)
\r
921 boolean tst1 = Character.isUpperCase(check1Parts[i].charAt(0));
\r
922 boolean tst2 = Character.isUpperCase(check2Parts[i].charAt(0));
\r
923 boolean tst3 = Character.isUpperCase(check3Parts[i].charAt(0));
\r
924 if (tst1 && !tst2 && !tst3)
\r
926 if (!tst1 && tst2 && !tst3)
\r
928 if (!tst1 && !tst2 && tst3)
\r
934 private boolean specialCharIsBetweenLetters(String conversionCheck)
\r
936 boolean bewteenLetters = true;
\r
937 for (int i = 0; i < conversionCheck.length(); i++)
\r
939 int charCode = (int)(conversionCheck.charAt(i));
\r
940 if (charCode > 0x7f)
\r
942 bewteenLetters = false;
\r
943 if (i > 0 && Character.isLetter((int)(conversionCheck.charAt(i-1))) ||
\r
944 (i < conversionCheck.length()-1 && Character.isLetter((int)(conversionCheck.charAt(i+1)))))
\r
946 bewteenLetters = true;
\r
951 return(bewteenLetters);
\r
954 private int arrayContainsAt(byte[] byteArray, int ft)
\r
956 for (int i = 0; i < byteArray.length; i++)
\r
958 if (byteArray[i] == (byte)ft) return(i);
\r
963 private DataField parseDataField(String tag, byte[] field) throws IOException
\r
967 errors.setRecordID(record.getControlNumber());
\r
968 errors.setCurrentField(tag);
\r
969 errors.setCurrentSubfield("n/a");
\r
970 cleanupBadFieldSeperators(field);
\r
972 ByteArrayInputStream bais = new ByteArrayInputStream(field);
\r
973 char ind1 = (char) bais.read();
\r
974 char ind2 = (char) bais.read();
\r
976 DataField dataField = factory.newDataField();
\r
977 dataField.setTag(tag);
\r
978 dataField.setIndicator1(ind1);
\r
979 dataField.setIndicator2(ind2);
\r
987 readByte = bais.read();
\r
990 switch (readByte) {
\r
992 code = bais.read();
\r
994 throw new IOException("unexpected end of data field");
\r
995 if (code == Constants.FT)
\r
997 size = getSubfieldLength(bais);
\r
998 data = new byte[size];
\r
1000 subfield = factory.newSubfield();
\r
1001 if (permissive) errors.setCurrentSubfield("" + (char)code);
\r
1002 String dataAsString = getDataAsString(data);
\r
1003 if (permissive && code == Constants.US)
\r
1006 dataAsString = dataAsString.substring(1);
\r
1007 errors.addError(ErrorHandler.MAJOR_ERROR,
\r
1008 "Subfield tag is a subfield separator, using first character of field as subfield tag.");
\r
1010 subfield.setCode((char) code);
\r
1011 subfield.setData(dataAsString);
\r
1012 dataField.addSubfield(subfield);
\r
1014 case Constants.FT:
\r
1021 static AnselToUnicode conv = null;
\r
1023 private void cleanupBadFieldSeperators(byte[] field)
\r
1025 if (conv == null) conv = new AnselToUnicode(true);
\r
1026 boolean hasEsc = false;
\r
1027 boolean inMultiByte = false;
\r
1028 boolean justCleaned = false;
\r
1031 for (int i = 0 ; i < field.length-1; i++)
\r
1033 if (field[i] == 0x1B)
\r
1036 if ("(,)-'".indexOf((char)field[i+1]) != -1)
\r
1038 inMultiByte = false;
\r
1040 else if (i + 2 < field.length && field[i+1] == '$' && field[i+2] == '1')
\r
1042 inMultiByte = true;
\r
1045 else if (i + 3 < field.length && (field[i+1] == '$' || field[i+2] == '$')&& ( field[i+2] == '1' || field[i+3] == '1'))
\r
1047 inMultiByte = true;
\r
1052 else if (inMultiByte && field[i] != 0x20) mbOffset = ( mbOffset == 0) ? 2 : mbOffset - 1;
\r
1053 if (inMultiByte && mbOffset == 0 && i + 2 < field.length)
\r
1056 byte f1 = field[i];
\r
1057 byte f2 = field[i+1] == 0x20 ? field[i+2] : field[i+1];
\r
1058 byte f3 = (field[i+1] == 0x20 || field[i+2] == 0x20) ? field[i+3] : field[i+2];
\r
1059 c = conv.getMBChar(conv.makeMultibyte((char)((f1 == Constants.US) ? 0x7C : f1),
\r
1060 (char)((f2 == Constants.US) ? 0x7C : f2),
\r
1061 (char)((f3 == Constants.US) ? 0x7C : f3)));
\r
1062 if (c == 0 && !justCleaned)
\r
1064 errors.addError(ErrorHandler.MAJOR_ERROR,
\r
1065 "Bad Multibyte character found, reinterpreting data as non-multibyte data");
\r
1066 inMultiByte = false;
\r
1068 else if (c == 0 && justCleaned)
\r
1070 c = conv.getMBChar(conv.makeMultibyte('!',(char)((f2 == Constants.US) ? 0x7C : f2),
\r
1071 (char)((f3 == Constants.US) ? 0x7C : f3)));
\r
1074 errors.addError(ErrorHandler.MAJOR_ERROR,
\r
1075 "Bad Multibyte character found, reinterpreting data as non-multibyte data");
\r
1076 inMultiByte = false;
\r
1080 errors.addError(ErrorHandler.MAJOR_ERROR,
\r
1081 "Character after restored vertical bar character makes bad multibyte character, changing it to \"!\"");
\r
1086 justCleaned = false;
\r
1087 if (field[i] == Constants.US )
\r
1089 if (inMultiByte && mbOffset != 0)
\r
1092 errors.addError(ErrorHandler.MAJOR_ERROR,
\r
1093 "Subfield separator found in middle of a multibyte character, changing it to a vertical bar, and continuing");
\r
1094 if (field[i+1] == '0')
\r
1096 if (field[i+2] == '(' && field[i+3] == 'B' )
\r
1098 field[i+1] = 0x1B;
\r
1099 errors.addError(ErrorHandler.MAJOR_ERROR,
\r
1100 "Character after restored vertical bar character makes bad multibyte character, changing it to ESC");
\r
1104 field[i+1] = 0x21;
\r
1105 errors.addError(ErrorHandler.MAJOR_ERROR,
\r
1106 "Character after restored vertical bar character makes bad multibyte character, changing it to \"!\"");
\r
1109 justCleaned = true;
\r
1111 else if (hasEsc && !((field[i+1] >= 'a' && field[i+1] <= 'z') || (field[i+1] >= '0' && field[i+1] <= '9')))
\r
1113 errors.addError(ErrorHandler.MAJOR_ERROR,
\r
1114 "Subfield separator followed by invalid subfield tag, changing separator to a vertical bar, and continuing");
\r
1116 justCleaned = true;
\r
1118 else if (hasEsc && i < field.length-3 &&
\r
1119 (field[i+1] == '0' && field[i+2] == '(' && field[i+3] == 'B' ))
\r
1121 errors.addError(ErrorHandler.MAJOR_ERROR,
\r
1122 "Subfield separator followed by invalid subfield tag, changing separator to a vertical bar, and continuing");
\r
1124 field[i+1] = 0x1B;
\r
1125 justCleaned = true;
\r
1127 else if (hasEsc && (field[i+1] == '0' ))
\r
1129 errors.addError(ErrorHandler.MAJOR_ERROR,
\r
1130 "Subfield separator followed by invalid subfield tag, changing separator to a vertical bar, and continuing");
\r
1132 field[i+1] = 0x21;
\r
1133 justCleaned = true;
\r
1135 else if (field[i+1] == Constants.US && field[i+2] == Constants.US )
\r
1137 errors.addError(ErrorHandler.MAJOR_ERROR,
\r
1138 "Three consecutive subfield separators, changing first two to vertical bars.");
\r
1140 field[i+1] = 0x7C;
\r
1141 justCleaned = true;
\r
1147 private int getFieldLength(DataInputStream bais) throws IOException
\r
1150 int bytesRead = 0;
\r
1152 switch (bais.read()) {
\r
1153 case Constants.FT:
\r
1160 errors.addError(ErrorHandler.MINOR_ERROR,
\r
1161 "Field not terminated trying to continue");
\r
1162 return (bytesRead);
\r
1165 throw new IOException("Field not terminated");
\r
1166 case Constants.US:
\r
1173 private int getSubfieldLength(ByteArrayInputStream bais) throws IOException {
\r
1175 int bytesRead = 0;
\r
1177 switch (bais.read()) {
\r
1178 case Constants.FT:
\r
1181 case Constants.US:
\r
1188 errors.addError(ErrorHandler.MINOR_ERROR, "Subfield not terminated trying to continue");
\r
1189 return (bytesRead);
\r
1192 throw new IOException("subfield not terminated");
\r
1199 private int parseRecordLength(byte[] leaderData) throws IOException {
\r
1200 InputStreamReader isr = new InputStreamReader(new ByteArrayInputStream(
\r
1203 char[] tmp = new char[5];
\r
1206 length = Integer.parseInt(new String(tmp));
\r
1207 } catch (NumberFormatException e) {
\r
1208 errors.addError(ErrorHandler.FATAL,
\r
1209 "Unable to parse record length, Unable to Continue");
\r
1210 throw new MarcException("unable to parse record length", e);
\r
1215 private void parseLeader(Leader ldr, byte[] leaderData) throws IOException {
\r
1216 InputStreamReader isr = new InputStreamReader(new ByteArrayInputStream(
\r
1218 char[] tmp = new char[5];
\r
1220 // Skip over bytes for record length, If we get here, its already been computed.
\r
1221 ldr.setRecordStatus((char) isr.read());
\r
1222 ldr.setTypeOfRecord((char) isr.read());
\r
1223 tmp = new char[2];
\r
1225 ldr.setImplDefined1(tmp);
\r
1226 ldr.setCharCodingScheme((char) isr.read());
\r
1227 char indicatorCount = (char) isr.read();
\r
1228 char subfieldCodeLength = (char) isr.read();
\r
1229 char baseAddr[] = new char[5];
\r
1230 isr.read(baseAddr);
\r
1231 tmp = new char[3];
\r
1233 ldr.setImplDefined2(tmp);
\r
1234 tmp = new char[4];
\r
1236 ldr.setEntryMap(tmp);
\r
1239 ldr.setIndicatorCount(Integer.parseInt(String.valueOf(indicatorCount)));
\r
1240 } catch (NumberFormatException e) {
\r
1241 throw new MarcException("unable to parse indicator count", e);
\r
1244 ldr.setSubfieldCodeLength(Integer.parseInt(String
\r
1245 .valueOf(subfieldCodeLength)));
\r
1246 } catch (NumberFormatException e) {
\r
1247 throw new MarcException("unable to parse subfield code length", e);
\r
1250 ldr.setBaseAddressOfData(Integer.parseInt(new String(baseAddr)));
\r
1251 } catch (NumberFormatException e) {
\r
1252 throw new MarcException("unable to parse base address of data", e);
\r
1257 private String getDataAsString(byte[] bytes)
\r
1259 String dataElement = null;
\r
1260 if (encoding.equals("UTF-8") || encoding.equals("UTF8"))
\r
1263 dataElement = new String(bytes, "UTF-8");
\r
1265 catch (UnsupportedEncodingException e) {
\r
1266 throw new MarcException("unsupported encoding", e);
\r
1269 else if (encoding.equals("UTF8-Maybe"))
\r
1272 dataElement = new String(bytes, "UTF-8");
\r
1274 catch (UnsupportedEncodingException e) {
\r
1275 throw new MarcException("unsupported encoding", e);
\r
1278 else if (encoding.equals("MARC-8") || encoding.equals("MARC8"))
\r
1280 dataElement = getMarc8Conversion(bytes);
\r
1282 else if (encoding.equalsIgnoreCase("Unimarc") || encoding.equals("IS05426"))
\r
1284 dataElement = getUnimarcConversion(bytes);
\r
1286 else if (encoding.equals("MARC8-Maybe"))
\r
1288 String dataElement1 = getMarc8Conversion(bytes);
\r
1289 String dataElement2 = getUnimarcConversion(bytes);
\r
1290 String dataElement3 = null;
\r
1293 dataElement3 = new String(bytes, "ISO-8859-1");
\r
1295 catch (UnsupportedEncodingException e)
\r
1297 // TODO Auto-generated catch block
\r
1298 e.printStackTrace();
\r
1300 if (dataElement1.equals(dataElement2) && dataElement1.equals(dataElement3))
\r
1302 dataElement = dataElement1;
\r
1306 conversionCheck1 = conversionCheck1 + "|>" + Normalizer.compose(dataElement1, false);
\r
1307 conversionCheck2 = conversionCheck2 + "|>" + dataElement2;
\r
1308 conversionCheck3 = conversionCheck3 + "|>" + dataElement3;
\r
1309 dataElement = dataElement1 + "%%@%%" + dataElement2 + "%%@%%" + dataElement3;
\r
1312 else if (encoding.equals("MARC8-Broken"))
\r
1316 dataElement = new String(bytes, "ISO-8859-1");
\r
1318 catch (UnsupportedEncodingException e)
\r
1320 // TODO Auto-generated catch block
\r
1321 e.printStackTrace();
\r
1323 String newdataElement = dataElement.replaceAll("<", "<");
\r
1324 newdataElement = newdataElement.replaceAll(">", ">");
\r
1325 newdataElement = newdataElement.replaceAll("&", "&");
\r
1326 newdataElement = newdataElement.replaceAll("'", "'");
\r
1327 newdataElement = newdataElement.replaceAll(""", "\"");
\r
1328 if (!newdataElement.equals(dataElement))
\r
1330 dataElement = newdataElement;
\r
1331 errors.addError(ErrorHandler.ERROR_TYPO, "Subfield contains escaped html character entities, un-escaping them. ");
\r
1333 String rep1 = ""+(char)0x1b+"\\$1$1";
\r
1334 String rep2 = ""+(char)0x1b+"\\(B";
\r
1335 newdataElement = dataElement.replaceAll("\\$1(.)", rep1);
\r
1336 newdataElement = newdataElement.replaceAll("\\(B", rep2);
\r
1337 if (!newdataElement.equals(dataElement))
\r
1339 dataElement = newdataElement;
\r
1340 errors.addError(ErrorHandler.MAJOR_ERROR, "Subfield seems to be missing MARC8 escape sequences, trying to restore them.");
\r
1344 dataElement = getMarc8Conversion(dataElement.getBytes("ISO-8859-1"));
\r
1346 catch (UnsupportedEncodingException e)
\r
1348 // TODO Auto-generated catch block
\r
1349 e.printStackTrace();
\r
1353 else if (encoding.equals("ISO-8859-1") || encoding.equals("ISO8859_1"))
\r
1356 dataElement = new String(bytes, "ISO-8859-1");
\r
1358 catch (UnsupportedEncodingException e) {
\r
1359 throw new MarcException("unsupported encoding", e);
\r
1364 throw new MarcException("Unknown or unsupported Marc character encoding:" + encoding);
\r
1366 if (errors != null && dataElement.matches("[^&]*&[a-z]*;.*"))
\r
1368 String newdataElement = dataElement.replaceAll("<", "<");
\r
1369 newdataElement = newdataElement.replaceAll(">", ">");
\r
1370 newdataElement = newdataElement.replaceAll("&", "&");
\r
1371 newdataElement = newdataElement.replaceAll("'", "'");
\r
1372 newdataElement = newdataElement.replaceAll(""", "\"");
\r
1373 if (!newdataElement.equals(dataElement))
\r
1375 dataElement = newdataElement;
\r
1376 errors.addError(ErrorHandler.ERROR_TYPO, "Subfield contains escaped html character entities, un-escaping them. ");
\r
1379 return dataElement;
\r
1382 private boolean byteArrayContains(byte[] bytes, byte[] seq)
\r
1384 for ( int i = 0; i < bytes.length - seq.length; i++)
\r
1386 if (bytes[i] == seq[0])
\r
1388 for (int j = 0; j < seq.length; j++)
\r
1390 if (bytes[i+j] != seq[j])
\r
1394 if (j == seq.length-1) return(true);
\r
1401 static byte badEsc[] = { (byte)('b'), (byte)('-'), 0x1b, (byte)('s') };
\r
1402 static byte overbar[] = { (byte)(char)(0xaf) };
\r
1404 private String getMarc8Conversion(byte[] bytes)
\r
1406 String dataElement = null;
\r
1407 if (converterAnsel == null) converterAnsel = new AnselToUnicode(errors);
\r
1408 if (permissive && (byteArrayContains(bytes, badEsc) || byteArrayContains(bytes, overbar)))
\r
1410 String newDataElement = null;
\r
1413 dataElement = new String(bytes, "ISO-8859-1");
\r
1414 newDataElement = dataElement.replaceAll("(\\e)b-\\es([psb])", "$1$2");
\r
1415 if (!newDataElement.equals(dataElement))
\r
1417 dataElement = newDataElement;
\r
1418 errors.addError(ErrorHandler.MINOR_ERROR, "Subfield contains odd pattern of subscript or superscript escapes. ");
\r
1420 newDataElement = dataElement.replace((char)0xaf, (char)0xe5);
\r
1421 if (!newDataElement.equals(dataElement))
\r
1423 dataElement = newDataElement;
\r
1424 errors.addError(ErrorHandler.ERROR_TYPO, "Subfield contains 0xaf overbar character, changing it to proper MARC8 representation ");
\r
1426 dataElement = converterAnsel.convert(dataElement);
\r
1428 catch (UnsupportedEncodingException e)
\r
1430 // TODO Auto-generated catch block
\r
1431 e.printStackTrace();
\r
1436 dataElement = converterAnsel.convert(bytes);
\r
1438 if (permissive && dataElement.matches("[^&]*&#x[0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f];.*"))
\r
1440 Pattern pattern = Pattern.compile("&#x([0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f]);");
\r
1441 Matcher matcher = pattern.matcher(dataElement);
\r
1442 StringBuffer newElement = new StringBuffer();
\r
1444 while (matcher.find())
\r
1446 newElement.append(dataElement.substring(prevEnd, matcher.start()));
\r
1447 newElement.append(getChar(matcher.group(1)));
\r
1448 prevEnd = matcher.end();
\r
1450 newElement.append(dataElement.substring(prevEnd));
\r
1451 dataElement = newElement.toString();
\r
1453 return(dataElement);
\r
1456 private String getUnimarcConversion(byte[] bytes)
\r
1458 if (converterUnimarc == null) converterUnimarc = new Iso5426ToUnicode();
\r
1459 String dataElement = converterUnimarc.convert(bytes);
\r
1460 dataElement = dataElement.replaceAll("\u0088", "");
\r
1461 dataElement = dataElement.replaceAll("\u0089", "");
\r
1462 // for ( int i = 0 ; i < bytes.length; i++)
\r
1464 // if (bytes[i] == -120 || bytes[i] == -119)
\r
1466 // char tmp = (char)bytes[i];
\r
1467 // char temp2 = dataElement.charAt(0);
\r
1468 // char temp3 = dataElement.charAt(4);
\r
1469 // int tmpi = (int)tmp;
\r
1470 // int tmp2 = (int)temp2;
\r
1471 // int tmp3 = (int)temp3;
\r
1476 if (dataElement.matches("[^<]*<U[+][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f]>.*"))
\r
1478 Pattern pattern = Pattern.compile("<U[+]([0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f])>");
\r
1479 Matcher matcher = pattern.matcher(dataElement);
\r
1480 StringBuffer newElement = new StringBuffer();
\r
1482 while (matcher.find())
\r
1484 newElement.append(dataElement.substring(prevEnd, matcher.start()));
\r
1485 newElement.append(getChar(matcher.group(1)));
\r
1486 prevEnd = matcher.end();
\r
1488 newElement.append(dataElement.substring(prevEnd));
\r
1489 dataElement = newElement.toString();
\r
1491 return(dataElement);
\r
1495 private String getChar(String charCodePoint)
\r
1497 int charNum = Integer.parseInt(charCodePoint, 16);
\r
1498 String result = ""+((char)charNum);
\r