Skip to content

Commit b1879bb

Browse files
pim97pim97
committed
Add axios/fetch drop-in replacement adapters - v2.0.1
- Add axios-compatible adapter (scrappey-wrapper/axios) - Add fetch-compatible adapter (scrappey-wrapper/fetch) - Full option mapping from axios/fetch to Scrappey options - Response transformation to match axios/fetch format - Global configuration support (defaults, configure) - TypeScript definitions for both adapters - Comprehensive test suite (13 tests, all passing) - Updated README with drop-in replacement documentation - Added example files for axios and fetch usage
1 parent e6d9a04 commit b1879bb

File tree

11 files changed

+1550
-6
lines changed

11 files changed

+1550
-6
lines changed

‎axios.js‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
/**
2+
* Axios-Compatible Adapter Entry Point
3+
*
4+
* Drop-in replacement for axios using Scrappey.
5+
*
6+
* Usage:
7+
* import axios from 'scrappey-wrapper/axios';
8+
* axios.defaults.apiKey = 'YOUR_API_KEY';
9+
* const response = await axios.get('https://example.com');
10+
*/
11+
12+
module.exports = require('./src/adapters/axios');

‎examples/nodejs/axios-example.js‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
/**
2+
* Scrappey - Axios Drop-in Replacement Example
3+
*
4+
* This example shows how to use Scrappey as a drop-in replacement for axios.
5+
* Just change your import and all requests automatically go through Scrappey!
6+
*/
7+
8+
// Instead of: const axios = require('axios');
9+
const axios = require('scrappey-wrapper/axios');
10+
11+
const API_KEY = process.env.SCRAPPEY_API_KEY || 'YOUR_API_KEY_HERE';
12+
13+
// Configure once
14+
axios.defaults.apiKey = API_KEY;
15+
axios.defaults.cloudflareBypass = true; // Enable Cloudflare bypass globally
16+
axios.defaults.premiumProxy = true; // Use premium proxies
17+
18+
async function basicExample() {
19+
console.log('\n=== Basic Axios Usage ===\n');
20+
21+
// Works exactly like axios!
22+
const response = await axios.get('https://httpbin.rs/get', {
23+
headers: {
24+
'User-Agent': 'MyApp/1.0'
25+
},
26+
params: {
27+
page: 1,
28+
limit: 10
29+
}
30+
});
31+
32+
console.log('Status:', response.status);
33+
console.log('Data:', response.data);
34+
console.log('Headers:', response.headers);
35+
36+
// Additional Scrappey data
37+
console.log('Verified:', response.scrappey?.verified);
38+
console.log('Time Elapsed:', response.scrappey?.timeElapsed, 'ms');
39+
}
40+
41+
async function postExample() {
42+
console.log('\n=== POST Request ===\n');
43+
44+
const response = await axios.post('https://httpbin.rs/post', {
45+
name: 'John Doe',
46+
email: '[email protected]'
47+
}, {
48+
headers: {
49+
'Content-Type': 'application/json'
50+
}
51+
});
52+
53+
console.log('Status:', response.status);
54+
console.log('Response:', response.data);
55+
}
56+
57+
async function sessionExample() {
58+
console.log('\n=== Session Management ===\n');
59+
60+
// Create a session
61+
const sessionResponse = await axios.createSession();
62+
const sessionId = sessionResponse.session;
63+
console.log('Created session:', sessionId);
64+
65+
// Use session for requests (cookies persist)
66+
const response1 = await axios.get('https://httpbin.rs/cookies/set/token/abc123', {
67+
session: sessionId
68+
});
69+
console.log('Set cookie, status:', response1.status);
70+
71+
// Second request - cookies persist
72+
const response2 = await axios.get('https://httpbin.rs/cookies', {
73+
session: sessionId
74+
});
75+
console.log('Cookies:', response2.data);
76+
77+
// Destroy session
78+
await axios.destroySession(sessionId);
79+
console.log('Session destroyed');
80+
}
81+
82+
async function cloudflareBypassExample() {
83+
console.log('\n=== Cloudflare Bypass ===\n');
84+
85+
// Automatically bypasses Cloudflare!
86+
const response = await axios.get('https://nowsecure.nl', {
87+
cloudflareBypass: true,
88+
premiumProxy: true
89+
});
90+
91+
console.log('Status:', response.status);
92+
console.log('Verified:', response.scrappey?.verified);
93+
console.log('Status Code:', response.status);
94+
}
95+
96+
async function customInstanceExample() {
97+
console.log('\n=== Custom Axios Instance ===\n');
98+
99+
// Create a custom instance with specific config
100+
const scrappeyAxios = axios.create({
101+
apiKey: API_KEY,
102+
cloudflareBypass: true,
103+
premiumProxy: true,
104+
proxyCountry: 'UnitedStates',
105+
automaticallySolveCaptchas: true,
106+
timeout: 60000
107+
});
108+
109+
const response = await scrappeyAxios.get('https://example.com');
110+
console.log('Status:', response.status);
111+
}
112+
113+
async function jsonResponseExample() {
114+
console.log('\n=== JSON Response ===\n');
115+
116+
const response = await axios.get('https://httpbin.rs/json', {
117+
responseType: 'json' // Automatically parses JSON
118+
});
119+
120+
console.log('Status:', response.status);
121+
console.log('Parsed JSON:', response.data);
122+
}
123+
124+
// Main execution
125+
async function main() {
126+
try {
127+
await basicExample();
128+
await postExample();
129+
await sessionExample();
130+
await jsonResponseExample();
131+
await customInstanceExample();
132+
133+
// Uncomment to test Cloudflare bypass (may use more credits)
134+
// await cloudflareBypassExample();
135+
136+
console.log('\n✓ All examples completed successfully!\n');
137+
} catch (error) {
138+
console.error('Error:', error.message);
139+
if (error.response) {
140+
console.error('Response:', error.response.data);
141+
}
142+
process.exit(1);
143+
}
144+
}
145+
146+
main();

‎examples/nodejs/fetch-example.js‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
/**
2+
* Scrappey - Fetch Drop-in Replacement Example
3+
*
4+
* This example shows how to use Scrappey as a drop-in replacement for fetch.
5+
* Just change your import and all requests automatically go through Scrappey!
6+
*/
7+
8+
// Instead of: const fetch = require('node-fetch');
9+
const fetch = require('scrappey-wrapper/fetch');
10+
11+
const API_KEY = process.env.SCRAPPEY_API_KEY || 'YOUR_API_KEY_HERE';
12+
13+
// Configure once
14+
fetch.configure({
15+
apiKey: API_KEY,
16+
cloudflareBypass: true, // Enable Cloudflare bypass globally
17+
premiumProxy: true // Use premium proxies
18+
});
19+
20+
async function basicExample() {
21+
console.log('\n=== Basic Fetch Usage ===\n');
22+
23+
// Works exactly like fetch!
24+
const response = await fetch('https://httpbin.rs/get', {
25+
headers: {
26+
'User-Agent': 'MyApp/1.0'
27+
}
28+
});
29+
30+
console.log('Status:', response.status);
31+
console.log('OK:', response.ok);
32+
console.log('Headers:', Object.fromEntries(response.headers.entries()));
33+
34+
const data = await response.json();
35+
console.log('Data:', data);
36+
37+
// Additional Scrappey data
38+
console.log('Verified:', response.scrappey?.verified);
39+
console.log('Time Elapsed:', response.scrappey?.timeElapsed, 'ms');
40+
}
41+
42+
async function postExample() {
43+
console.log('\n=== POST Request ===\n');
44+
45+
const response = await fetch('https://httpbin.rs/post', {
46+
method: 'POST',
47+
headers: {
48+
'Content-Type': 'application/json'
49+
},
50+
body: JSON.stringify({
51+
name: 'John Doe',
52+
email: '[email protected]'
53+
})
54+
});
55+
56+
console.log('Status:', response.status);
57+
const data = await response.json();
58+
console.log('Response:', data);
59+
}
60+
61+
async function sessionExample() {
62+
console.log('\n=== Session Management ===\n');
63+
64+
// Create a session
65+
const sessionResponse = await fetch.createSession();
66+
const sessionId = sessionResponse.session;
67+
console.log('Created session:', sessionId);
68+
69+
// Use session for requests (cookies persist)
70+
const response1 = await fetch('https://httpbin.rs/cookies/set/token/abc123', {
71+
session: sessionId
72+
});
73+
console.log('Set cookie, status:', response1.status);
74+
75+
// Second request - cookies persist
76+
const response2 = await fetch('https://httpbin.rs/cookies', {
77+
session: sessionId
78+
});
79+
const cookies = await response2.text();
80+
console.log('Cookies:', cookies);
81+
82+
// Destroy session
83+
await fetch.destroySession(sessionId);
84+
console.log('Session destroyed');
85+
}
86+
87+
async function cloudflareBypassExample() {
88+
console.log('\n=== Cloudflare Bypass ===\n');
89+
90+
// Automatically bypasses Cloudflare!
91+
const response = await fetch('https://nowsecure.nl', {
92+
cloudflareBypass: true,
93+
premiumProxy: true
94+
});
95+
96+
console.log('Status:', response.status);
97+
console.log('OK:', response.ok);
98+
console.log('Verified:', response.scrappey?.verified);
99+
}
100+
101+
async function responseMethodsExample() {
102+
console.log('\n=== Response Methods ===\n');
103+
104+
const response = await fetch('https://httpbin.rs/json');
105+
106+
// All standard fetch response methods work
107+
console.log('Status:', response.status);
108+
console.log('Status Text:', response.statusText);
109+
console.log('Headers:', Object.fromEntries(response.headers.entries()));
110+
111+
// Parse as JSON
112+
const json = await response.json();
113+
console.log('JSON:', json);
114+
115+
// Or get as text
116+
// const text = await response.text();
117+
// console.log('Text:', text);
118+
}
119+
120+
async function errorHandlingExample() {
121+
console.log('\n=== Error Handling ===\n');
122+
123+
try {
124+
const response = await fetch('https://httpbin.rs/status/404');
125+
126+
if (!response.ok) {
127+
throw new Error(`HTTP error! status: ${response.status}`);
128+
}
129+
} catch (error) {
130+
console.error('Error:', error.message);
131+
}
132+
}
133+
134+
// Main execution
135+
async function main() {
136+
try {
137+
await basicExample();
138+
await postExample();
139+
await sessionExample();
140+
await responseMethodsExample();
141+
await errorHandlingExample();
142+
143+
// Uncomment to test Cloudflare bypass (may use more credits)
144+
// await cloudflareBypassExample();
145+
146+
console.log('\n✓ All examples completed successfully!\n');
147+
} catch (error) {
148+
console.error('Error:', error.message);
149+
process.exit(1);
150+
}
151+
}
152+
153+
main();

‎fetch.js‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
/**
2+
* Fetch-Compatible Adapter Entry Point
3+
*
4+
* Drop-in replacement for fetch using Scrappey.
5+
*
6+
* Usage:
7+
* import fetch from 'scrappey-wrapper/fetch';
8+
* fetch.configure({ apiKey: 'YOUR_API_KEY' });
9+
* const response = await fetch('https://example.com');
10+
*/
11+
12+
module.exports = require('./src/adapters/fetch');

‎package.json‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,33 @@
11
{
22
"name": "scrappey-wrapper",
3-
"version": "2.0.0",
3+
"version": "2.0.1",
44
"description": "Official Node.js wrapper for the Scrappey web scraping API. Bypass Cloudflare, Datadome, PerimeterX, and other antibot protections. Solve captchas automatically.",
55
"main": "index.js",
66
"types": "types/index.d.ts",
77
"exports": {
88
".": {
99
"require": "./index.js",
1010
"types": "./types/index.d.ts"
11+
},
12+
"./axios": {
13+
"require": "./axios.js",
14+
"types": "./types/adapters/axios.d.ts"
15+
},
16+
"./fetch": {
17+
"require": "./fetch.js",
18+
"types": "./types/adapters/fetch.d.ts"
1119
}
1220
},
1321
"files": [
1422
"index.js",
23+
"axios.js",
24+
"fetch.js",
1525
"src/",
16-
"types/"
26+
"types/",
27+
"tests/"
1728
],
1829
"scripts": {
19-
"test": "echo \"Tests passing\" && exit 0"
30+
"test": "node tests/adapters.test.js"
2031
},
2132
"repository": {
2233
"type": "git",

0 commit comments

Comments
 (0)