| 1234567891011121314151617181920212223242526272829303132333435363738394041 | 
							- // \n = 10
 
- // ; = 59
 
- // { = 123
 
- // } = 125
 
- // <space> = 32
 
- // \r = 13
 
- // \t = 9
 
- const splitIntoPotentialTokens = str => {
 
- 	const len = str.length;
 
- 	if (len === 0) return null;
 
- 	const results = [];
 
- 	let i = 0;
 
- 	for (; i < len; ) {
 
- 		const s = i;
 
- 		block: {
 
- 			let cc = str.charCodeAt(i);
 
- 			while (cc !== 10 && cc !== 59 && cc !== 123 && cc !== 125) {
 
- 				if (++i >= len) break block;
 
- 				cc = str.charCodeAt(i);
 
- 			}
 
- 			while (
 
- 				cc === 59 ||
 
- 				cc === 32 ||
 
- 				cc === 123 ||
 
- 				cc === 125 ||
 
- 				cc === 13 ||
 
- 				cc === 9
 
- 			) {
 
- 				if (++i >= len) break block;
 
- 				cc = str.charCodeAt(i);
 
- 			}
 
- 			if (cc === 10) {
 
- 				i++;
 
- 			}
 
- 		}
 
- 		results.push(str.slice(s, i));
 
- 	}
 
- 	return results;
 
- };
 
- module.exports = splitIntoPotentialTokens;
 
 
  |