Skip to content

Commit cc74a84

Browse files
thiswillbeyourgithubaider-chat-bot
thiswillbeyourgithub
and
aider-chat-bot
committed
tests: chore: add shell-based CLI test script for more reliable testing
Co-authored-by: aider (openrouter/anthropic/claude-sonnet-4) <aider@aider.chat>
1 parent a801718 commit cc74a84

1 file changed

Lines changed: 288 additions & 0 deletions

File tree

‎tests/test_cli.sh‎

Lines changed: 288 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,288 @@
1+
#!/bin/zsh
2+
3+
set -e
4+
set -o pipefail
5+
set -u
6+
7+
# Environment setup for CLI tests (same as in pytest file)
8+
export OVERRIDE_USER_DIR_PYTEST_WDOC="true"
9+
export WDOC_TYPECHECKING="crash"
10+
11+
# Store the venv python path to ensure we use it consistently
12+
PYTHON_EXEC=$(which python)
13+
14+
# Colors for output
15+
RED='\033[0;31m'
16+
GREEN='\033[0;32m'
17+
YELLOW='\033[1;33m'
18+
NC='\033[0m' # No Color
19+
20+
# Test result tracking
21+
PASSED_TESTS=0
22+
FAILED_TESTS=0
23+
TOTAL_TESTS=0
24+
25+
# Helper function to run a test and track results
26+
run_test() {
27+
local test_name="$1"
28+
local test_function="$2"
29+
30+
TOTAL_TESTS=$((TOTAL_TESTS + 1))
31+
echo "${YELLOW}[TEST $TOTAL_TESTS]${NC} Running: $test_name"
32+
33+
if $test_function; then
34+
echo "${GREEN}[PASS]${NC} $test_name"
35+
PASSED_TESTS=$((PASSED_TESTS + 1))
36+
else
37+
echo "${RED}[FAIL]${NC} $test_name"
38+
FAILED_TESTS=$((FAILED_TESTS + 1))
39+
# Don't exit immediately, continue with other tests
40+
return 1
41+
fi
42+
}
43+
44+
# Test functions (converted from pytest)
45+
test_help_output_shell() {
46+
local output
47+
if ! output=$(wdoc --help 2>&1); then
48+
echo "FAIL: wdoc --help command failed"
49+
return 1
50+
fi
51+
52+
# Check that dynamic docstring placeholder is not present
53+
if echo "$output" | grep -q "This docstring is dynamically updated with the content of wdoc/docs/help.md"; then
54+
echo "FAIL: Found dynamic docstring placeholder in help output"
55+
return 1
56+
fi
57+
58+
# Check that actual content is present
59+
if ! echo "$output" | grep -q "Content of wdoc/docs/help.md"; then
60+
echo "FAIL: Did not find expected content in help output"
61+
return 1
62+
fi
63+
64+
return 0
65+
}
66+
67+
test_help_output_python() {
68+
local output
69+
if ! output=$($PYTHON_EXEC -m wdoc --help 2>&1); then
70+
echo "FAIL: python -m wdoc --help command failed"
71+
return 1
72+
fi
73+
74+
# Check that dynamic docstring placeholder is not present
75+
if echo "$output" | grep -q "This docstring is dynamically updated with the content of wdoc/docs/help.md"; then
76+
echo "FAIL: Found dynamic docstring placeholder in help output"
77+
return 1
78+
fi
79+
80+
# Check that actual content is present
81+
if ! echo "$output" | grep -q "Content of wdoc/docs/help.md"; then
82+
echo "FAIL: Did not find expected content in help output"
83+
return 1
84+
fi
85+
86+
return 0
87+
}
88+
89+
test_parse_doc_help_output_shell() {
90+
local output
91+
if ! output=$(wdoc parse --help 2>&1); then
92+
echo "FAIL: wdoc parse --help command failed"
93+
return 1
94+
fi
95+
96+
# Check that dynamic docstring placeholder is not present
97+
if echo "$output" | grep -q "This docstring is dynamically updated with the content of wdoc/docs/parse_doc_help.md"; then
98+
echo "FAIL: Found dynamic docstring placeholder in parse help output"
99+
return 1
100+
fi
101+
102+
# Check that actual content is present
103+
if ! echo "$output" | grep -q "Content of wdoc/docs/parse_doc_help.md"; then
104+
echo "FAIL: Did not find expected content in parse help output"
105+
return 1
106+
fi
107+
108+
return 0
109+
}
110+
111+
test_parse_doc_help_output_python() {
112+
local output
113+
if ! output=$($PYTHON_EXEC -m wdoc parse --help 2>&1); then
114+
echo "FAIL: python -m wdoc parse --help command failed"
115+
return 1
116+
fi
117+
118+
# Check that dynamic docstring placeholder is not present
119+
if echo "$output" | grep -q "This docstring is dynamically updated with the content of wdoc/docs/parse_doc_help.md"; then
120+
echo "FAIL: Found dynamic docstring placeholder in parse help output"
121+
return 1
122+
fi
123+
124+
# Check that actual content is present
125+
if ! echo "$output" | grep -q "Content of wdoc/docs/parse_doc_help.md"; then
126+
echo "FAIL: Did not find expected content in parse help output"
127+
return 1
128+
fi
129+
130+
return 0
131+
}
132+
133+
test_error_message_shell_debug() {
134+
local command="wdoc --task=summarize --path https://lemonde.fr/ --filetype=test --debug"
135+
local expected_substring="(Pdb) "
136+
local output
137+
138+
# Use timeout and expect it to timeout since debug mode waits for input
139+
# Capture both stdout and stderr, and handle the timeout exit code
140+
if output=$(timeout 30s bash -c "$command" 2>&1); then
141+
echo "FAIL: Command should have timed out but completed successfully"
142+
return 1
143+
fi
144+
145+
# timeout returns 124 on timeout, which is what we expect
146+
local exit_code=$?
147+
if [[ $exit_code -ne 124 ]]; then
148+
echo "FAIL: Expected timeout (exit code 124), got exit code $exit_code"
149+
return 1
150+
fi
151+
152+
# Check if we got the expected substring in the output
153+
if ! echo "$output" | grep -q "$expected_substring"; then
154+
echo "FAIL: Expected '$expected_substring' not found in command output"
155+
echo "Output was: $output"
156+
return 1
157+
fi
158+
159+
return 0
160+
}
161+
162+
test_get_piped_input_detection() {
163+
# Test text piping
164+
local input_text="This is test text.\nWith multiple lines."
165+
local cmd_text='import sys; from wdoc.utils.misc import get_piped_input; data = get_piped_input(); sys.stdout.write(data)'
166+
local result_text
167+
168+
if ! result_text=$(echo -e "$input_text" | $PYTHON_EXEC -c "$cmd_text" 2>&1); then
169+
echo "FAIL: Text piping command failed"
170+
return 1
171+
fi
172+
173+
if ! echo "$result_text" | grep -q "This is test text"; then
174+
echo "FAIL: Text piping did not work correctly"
175+
echo "Expected to find input text in output, got: $result_text"
176+
return 1
177+
fi
178+
179+
# Test binary piping (using printf to create binary-like data)
180+
local cmd_bytes='import sys; from wdoc.utils.misc import get_piped_input; data = get_piped_input(); sys.stdout.buffer.write(data)'
181+
local result_bytes
182+
183+
if ! result_bytes=$(printf '\x01\x02\x03\xffbinary data' | $PYTHON_EXEC -c "$cmd_bytes" 2>&1); then
184+
echo "FAIL: Binary piping command failed"
185+
return 1
186+
fi
187+
188+
if ! echo "$result_bytes" | grep -q "binary data"; then
189+
echo "FAIL: Binary piping did not work correctly"
190+
echo "Expected to find binary data in output, got: $result_bytes"
191+
return 1
192+
fi
193+
194+
return 0
195+
}
196+
197+
test_parse_nytimes_shell() {
198+
local output
199+
if ! output=$($PYTHON_EXEC -m wdoc parse "https://www.nytimes.com/" --format text 2>&1); then
200+
echo "FAIL: NYTimes parsing command failed"
201+
return 1
202+
fi
203+
204+
# Verify we got substantial content
205+
local output_length=${#output}
206+
if [[ $output_length -le 100 ]]; then
207+
echo "FAIL: Expected significant text content from NYTimes, got only $output_length characters"
208+
echo "Output was: $output"
209+
return 1
210+
fi
211+
212+
echo "INFO: Got $output_length characters from NYTimes"
213+
return 0
214+
}
215+
216+
test_ddg_search_nvidia() {
217+
local output
218+
local cmd="$PYTHON_EXEC -m wdoc --task=query --path='How is Nvidia doing this month?' --query='How is Nvidia doing this month?' --filetype=ddg --ddg_max_result=3 --ddg_region=us-US --model=testing/testing --loading_failure=warn --oneoff --file_loader_parallel_backend=threading"
219+
220+
if ! output=$(timeout 120s bash -c "$cmd" 2>&1); then
221+
local exit_code=$?
222+
if [[ $exit_code -eq 124 ]]; then
223+
echo "FAIL: DDG search command timed out"
224+
else
225+
echo "FAIL: DDG search command failed with exit code $exit_code"
226+
fi
227+
echo "Output was: $output"
228+
return 1
229+
fi
230+
231+
# Check that we got some output
232+
local output_length=${#output}
233+
if [[ $output_length -le 100 ]]; then
234+
echo "FAIL: Expected substantial output from DDG search, got only $output_length characters"
235+
echo "Output was: $output"
236+
return 1
237+
fi
238+
239+
# Should contain the testing model's standard response
240+
if ! echo "$output" | grep -q "Lorem ipsum dolor sit amet"; then
241+
echo "FAIL: Output did not contain expected testing string"
242+
echo "Output was: $output"
243+
return 1
244+
fi
245+
246+
echo "INFO: Got $output_length characters from DDG search"
247+
return 0
248+
}
249+
250+
# Main execution
251+
echo "${YELLOW}Starting CLI tests...${NC}"
252+
echo "Using Python executable: $PYTHON_EXEC"
253+
echo
254+
255+
# Run basic tests
256+
echo "${YELLOW}=== BASIC TESTS ===${NC}"
257+
run_test "Help output (python)" test_help_output_python
258+
run_test "Help output (shell)" test_help_output_shell
259+
run_test "Parse doc help output (python)" test_parse_doc_help_output_python
260+
run_test "Parse doc help output (shell)" test_parse_doc_help_output_shell
261+
run_test "Error message in debug mode" test_error_message_shell_debug
262+
run_test "Piped input detection" test_get_piped_input_detection
263+
run_test "Parse NYTimes homepage" test_parse_nytimes_shell
264+
265+
echo
266+
echo "${YELLOW}=== API TESTS ===${NC}"
267+
echo "Note: API tests require external network access and may take longer"
268+
269+
# Check if we should run API tests (similar to pytest's -m api marker)
270+
if [[ "${1:-}" == "api" ]] || [[ "$*" == *"api"* ]]; then
271+
run_test "DuckDuckGo search functionality" test_ddg_search_nvidia
272+
else
273+
echo "${YELLOW}Skipping API tests. Use 'api' argument to run them.${NC}"
274+
fi
275+
276+
echo
277+
echo "${YELLOW}=== TEST SUMMARY ===${NC}"
278+
echo "Total tests: $TOTAL_TESTS"
279+
echo "${GREEN}Passed: $PASSED_TESTS${NC}"
280+
echo "${RED}Failed: $FAILED_TESTS${NC}"
281+
282+
if [[ $FAILED_TESTS -gt 0 ]]; then
283+
echo "${RED}Some tests failed!${NC}"
284+
exit 1
285+
else
286+
echo "${GREEN}All tests passed!${NC}"
287+
exit 0
288+
fi

0 commit comments

Comments
 (0)