From: Bonita.Montero@gmail.com
How long have you been dealing with that ? 10 days ?
I finished the AVX-512 version in C++ in two hours.
Am 16.12.2025 um 01:26 schrieb Michael Sanders:
> On Fri, 12 Dec 2025 15:33:01 -0800, Chris M. Thomasson wrote:
>
>> define the probability? Say in 0...1?
> Probabilities for Chris... Enjoy.
>
> #include
> #include
> #include
>
> /*
> * map_strict[]
> *
> * Valid bytes:
> * - ASCII printable: 0x20–0x7E
> * - ISO-8859-1 high printable: 0xA0–0xFF
> * - Whitespace: TAB (0x09), LF (0x0A), CR (0x0D)
> *
> * Invalid bytes (binary indicators):
> * - NULL byte (0x00)
> * - C0 controls (0x01–0x08, 0x0B–0x0C, 0x0E–0x1F)
> * - DEL (0x7F)
> * - C1 controls (0x80–0x9F)
> */
>
> static const uint8_t map_strict[256] = {
> 0,0,0,0,0,0,0,0,0,1,1,0,0,1,0,0, // 00
> 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 10
> 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 20
> 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 30
> 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 40
> 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 50
> 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 60
> 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0, // 70
> 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 80
> 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 90
> 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // A0
> 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // B0
> 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // C0
> 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // D0
> 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // E0
> 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1 // F0
> };
>
> /*
> * map_loose[]
> *
> * Valid bytes:
> * - ASCII printable characters: 0x20–0x7E
> * - Whitespace: TAB (0x09), LF (0x0A), CR (0x0D)
> * - High bytes: 0x80–0xFF
> *
> * Invalid bytes (binary indicators):
> * - NULL byte: 0x00
> * - C0 control codes: 0x01–0x08, 0x0B–0x0C, 0x0E–0x1F
> * - DEL character: 0x7F
> */
>
> static const uint8_t map_loose[256] = {
> 0,0,0,0,0,0,0,0,0,1,1,0,0,1,0,0, // 00
> 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 10
> 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 20
> 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 30
> 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 40
> 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 50
> 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 60
> 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 70
> 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 80
> 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 90
> 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // A0
> 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // B0
> 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // C0
> 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // D0
> 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // E0
> 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1 // F0
> };
>
> /*
> * is_text_file(const char *path, const uint8_t map[256], double
probability)
> *
> * Determines whether a file is likely a text file driven by a byte map
> * & probability threshold.
> *
> * Signature:
> *
> * path
> * Pathname of the file to examine. The function follows symlinks
> * & only operates on regular files.
> *
> * map[256]
> * A 256-entry lookup table indexed by byte value (0–255).
> * Each entry must be either:
> * 1 -> byte is considered valid text
> * 0 -> byte is considered invalid (binary indicator)
> *
> * probability
> * Maximum allowed percentage (0.0 to 100.0) of invalid bytes.
> * If the percentage of bytes for which map[byte] == 0 exceeds
> * this value, the file is classified as non-text.
> *
> * Behavior:
> * - Reads the file in binary mode.
> * - Counts total bytes & 'bad' bytes (map[] == 0).
> * - Returns 1 if bad byte percentage <= probability.
> * - Returns 0 if bad byte percentage > probability.
> * - Returns -1 on access/open errors or if the file is not regular.
> * - Function exits early if impossible to satisfy probability threshold.
> *
> * Returns:
> * 1 - text
> * 0 - binary indicator
> * -1 - could not open
> *
> * Example with 94.3% probability threshold:
> * is_text_file("./example.txt", your_map[256], 94.3)
> *
> */
>
> int is_text_file(const char *path, const uint8_t map[256], double
probability) {
>
> struct stat st;
> if (stat(path, &st) != 0) return -1; // can not access file
> if (!S_ISREG(st.st_mode)) return -1; // reject: directori
s/devices/sockets
> FILE *f = fopen(path, "rb"); if (!f) return -1; // could not open file
>
> // 4KB: 4096, 8KB: 8192, 16KB: 16384, 32KB: 32768, 64KB: 65536
> unsigned char buf[16384];
> size_t n, i;
> size_t total = 0;
> size_t bad = 0;
>
> while ((n = fread(buf, 1, sizeof(buf), f)) > 0) {
> for (i = 0; i < n; i++) {
> total++;
> if (!map[buf[i]]) bad++;
> if (((double)bad / total) * 100.0 > probability) {
> fclose(f);
> return 0; // not text
> }
> }
> }
>
> fclose(f);
>
> if (total == 0) return 0; // empty file or total binary...
>
> // final probability check
> return (((double)bad / total) * 100.0 <= probability) ? 1 : 0;
> }
>
--- SoupGate-Win32 v1.05
* Origin: you cannot sedate... all the things you hate (1:229/2)
|
