Utf8SkipScan.hpp

/*
 * Open Chinese Convert
 *
 * Copyright 2010-2026 Carbo Kuo and contributors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
 
#pragma once
 
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <vector>
 
#include "UTF8Util.hpp"
 
namespace opencc {
namespace internal {

inline uint32_t DecodeCodePoint23(const char* str, size_t charLength) {
  const unsigned char lead = static_cast<unsigned char>(str[0]);
  if (charLength == 2) {
    return ((lead & 0x1FU) << 6) |
           (static_cast<unsigned char>(str[1]) & 0x3FU);
  }
  return ((lead & 0x0FU) << 12) |
         ((static_cast<unsigned char>(str[1]) & 0x3FU) << 6) |
         (static_cast<unsigned char>(str[2]) & 0x3FU);
}

struct Utf8SkipTable {
  bool candidate[256] = {};
  bool asciiHasCandidates = false;
  std::vector<uint64_t> bmpCandidates;
 
  bool CharLevel() const { return !bmpCandidates.empty(); }
 
  void EnableCharLevel() { bmpCandidates.assign(0x10000 / 64, 0); }

  void DisableCharLevel() { bmpCandidates.clear(); }
 
  void MarkCharCandidate(uint32_t codePoint) {
    if (CharLevel() && codePoint < 0x10000) {
      bmpCandidates[codePoint >> 6] |= uint64_t(1) << (codePoint & 63);
    }
  }
 
  bool IsCharCandidate(uint32_t codePoint) const {
    assert(CharLevel());
    return (bmpCandidates[codePoint >> 6] >> (codePoint & 63)) & 1;
  }
 
  void MarkAllCandidates() {
    for (size_t b = 0; b < 256; b++) {
      candidate[b] = true;
    }
    DisableCharLevel();
  }

  void Finalize() {
    static_assert(UTF8Util::kFirstIdeographicDescriptionOperator >= 0x0800 &&
                      UTF8Util::kLastIdeographicDescriptionOperator <= 0xFFFF,
                  "IDS operator range must lie in the 3-byte UTF-8 block");
    for (uint32_t cp = UTF8Util::kFirstIdeographicDescriptionOperator;
         cp <= UTF8Util::kLastIdeographicDescriptionOperator; cp++) {
      if (UTF8Util::IdeographicDescriptionOperatorArity(cp) == 0) {
        continue;
      }
      // The operator range lies in the 3-byte UTF-8 block of the BMP.
      candidate[0xE0 | (cp >> 12)] = true;
      MarkCharCandidate(cp);
    }
    asciiHasCandidates = false;
    for (size_t b = 0; b < 0x80; b++) {
      if (candidate[b]) {
        asciiHasCandidates = true;
        break;
      }
    }
  }
};

inline size_t AsciiRunLength(const char* str, size_t len) {
  size_t pos = 0;
  for (; pos + sizeof(uint64_t) <= len; pos += sizeof(uint64_t)) {
    uint64_t word;
    std::memcpy(&word, str + pos, sizeof(word));
    if ((word & UINT64_C(0x8080808080808080)) != 0) {
      break;
    }
  }
  // Resolve the exact position within the word that stopped the loop (at
  // most 8 iterations), and handle the trailing partial word.
  for (; pos < len; pos++) {
    if (static_cast<unsigned char>(str[pos]) >= 0x80) {
      break;
    }
  }
  return pos;
}

inline size_t SkipNonCandidateBytes(const Utf8SkipTable& table, const char* str,
                                    size_t len) {
  size_t pos = 0;
  while (pos < len) {
    const unsigned char lead = static_cast<unsigned char>(str[pos]);
    if (lead < 0x80) {
      if (!table.asciiHasCandidates) {
        pos += AsciiRunLength(str + pos, len - pos);
        continue;
      }
      if (table.candidate[lead]) {
        break;
      }
      pos++;
      continue;
    }
    const size_t charLength = UTF8Util::NextCharLengthNoException(str + pos);
    if (charLength == 0 || charLength > len - pos) {
      break;
    }
    if (table.CharLevel() && (charLength == 2 || charLength == 3)) {
      // DecodeCodePoint23 does not validate continuation bytes; the table
      // builder decodes key bytes with the same function, so a position is
      // skipped only when its exact bytes cannot begin any key. Either way
      // the scan consumes the same charLength bytes the per-character path
      // would, so behavior stays equivalent even on malformed input.
      if (table.IsCharCandidate(DecodeCodePoint23(str + pos, charLength))) {
        break;
      }
    } else if (table.candidate[lead]) {
      break;
    }
    pos += charLength;
  }
  return pos;
}
 
} // namespace internal
} // namespace opencc