From f84d6fdc8553893bd70ffa4ce61ce6374296dc59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alexander=20W=C3=BChr?= <alex@pnyhf.eu>
Date: Thu, 19 Sep 2024 17:24:54 +0200
Subject: [PATCH] refactor: straighten token

---
 src/fortheck/token.gleam       | 27 +++++++++++++++++----
 test/fortheck/stack_test.gleam |  2 +-
 test/fortheck/token_test.gleam | 44 ++++++++++++++++++++++++++++++++++
 test/token_test.gleam          | 23 ------------------
 4 files changed, 68 insertions(+), 28 deletions(-)
 create mode 100644 test/fortheck/token_test.gleam
 delete mode 100644 test/token_test.gleam

diff --git a/src/fortheck/token.gleam b/src/fortheck/token.gleam
index 4db5920..70b851c 100644
--- a/src/fortheck/token.gleam
+++ b/src/fortheck/token.gleam
@@ -1,6 +1,6 @@
 import gleam/bool
 import gleam/int
-import gleam/iterator.{type Iterator}
+import gleam/iterator.{type Iterator, type Step, Done, Next}
 import gleam/result
 import gleam/string
 
@@ -9,6 +9,10 @@ pub type Token {
   Number(Int)
 }
 
+pub type Error {
+  UnexpectedEndOfTokens
+}
+
 pub fn from_string(token: String) -> Token {
   case int.parse(token) {
     Ok(n) -> Number(n)
@@ -23,14 +27,14 @@ fn is_whitespace(string: String) -> Bool {
   }
 }
 
-fn yield_token(acc: String, string: String) -> iterator.Step(Token, String) {
-  use <- bool.guard(when: acc == "" && string == "", return: iterator.Done)
+fn yield_token(acc: String, string: String) -> Step(Token, String) {
+  use <- bool.guard(when: acc == "" && string == "", return: Done)
 
   let #(char, rest) = string.pop_grapheme(string) |> result.unwrap(#("", ""))
 
   case is_whitespace(char), acc == "" {
     True, True -> yield_token("", rest)
-    True, _ -> iterator.Next(from_string(acc), rest)
+    True, _ -> Next(from_string(acc), rest)
     False, _ -> yield_token(acc <> char, rest)
   }
 }
@@ -38,3 +42,18 @@ fn yield_token(acc: String, string: String) -> iterator.Step(Token, String) {
 pub fn tokenize(string: String) -> Iterator(Token) {
   iterator.unfold(from: string, with: yield_token("", _))
 }
+
+pub fn skip_over(
+  in tokens: Iterator(Token),
+  this token: Token,
+) -> Result(Iterator(Token), Error) {
+  let tokens =
+    tokens
+    |> iterator.drop_while(fn(t) { t != token })
+    |> iterator.drop(1)
+
+  case iterator.first(tokens) {
+    Ok(_) -> Ok(tokens)
+    Error(_) -> Error(UnexpectedEndOfTokens)
+  }
+}
diff --git a/test/fortheck/stack_test.gleam b/test/fortheck/stack_test.gleam
index 1b17cce..4a490f7 100644
--- a/test/fortheck/stack_test.gleam
+++ b/test/fortheck/stack_test.gleam
@@ -1,4 +1,4 @@
-import fortheck/stack.{type Error, StackOverflow, StackUnderflow}
+import fortheck/stack.{StackOverflow, StackUnderflow}
 import gleam/iterator
 import gleam/pair
 import gleam/result
diff --git a/test/fortheck/token_test.gleam b/test/fortheck/token_test.gleam
new file mode 100644
index 0000000..97faa1f
--- /dev/null
+++ b/test/fortheck/token_test.gleam
@@ -0,0 +1,44 @@
+import fortheck/token.{UnexpectedEndOfTokens}
+import gleam/iterator
+import gleeunit
+import gleeunit/should
+
+pub fn main() {
+  gleeunit.main()
+}
+
+pub fn tokenize_test() {
+  let string = "3 4\nMUL\t5  \n\n \n  dIv"
+
+  string
+  |> token.tokenize
+  |> iterator.to_list
+  |> should.equal([
+    token.Number(3),
+    token.Number(4),
+    token.Word("MUL"),
+    token.Number(5),
+    token.Word("DIV"),
+  ])
+}
+
+pub fn skip_over_test() {
+  let string = "3 4\nMUL\t5  \n\n \n mul dIv"
+
+  string
+  |> token.tokenize
+  |> token.skip_over(token.Word("MUL"))
+  |> should.be_ok
+  |> iterator.to_list
+  |> should.equal([token.Number(5), token.Word("MUL"), token.Word("DIV")])
+}
+
+pub fn skip_over_eof_test() {
+  let string = "3 4\nMUL\t5  \n\n \n mul dIv"
+
+  string
+  |> token.tokenize
+  |> token.skip_over(token.Word("DIV"))
+  |> should.be_error
+  |> should.equal(UnexpectedEndOfTokens)
+}
diff --git a/test/token_test.gleam b/test/token_test.gleam
deleted file mode 100644
index 8199b19..0000000
--- a/test/token_test.gleam
+++ /dev/null
@@ -1,23 +0,0 @@
-import fortheck/token
-import gleam/iterator
-import gleeunit
-import gleeunit/should
-
-pub fn main() {
-  gleeunit.main()
-}
-
-pub fn tokenize_test() {
-  let string = "3 4\nMUL\t5  \n\n \n  dIv"
-
-  string
-  |> token.tokenize
-  |> iterator.to_list
-  |> should.equal([
-    token.Number(3),
-    token.Number(4),
-    token.Word("MUL"),
-    token.Number(5),
-    token.Word("DIV"),
-  ])
-}