Hi all. I have a Lambda question. I am trying to use the SERP API for some scholar data with a Lambda function. My function is working mostly for the first pull but am getting a connection refused for the follow up loop in my test. should i split up the function into multiple (or put an API gateway)?
It's the commented out line for const additionalcitations and getCitationsFromPaginationUrls that is getting the connection refused. Why is it refusing connection on the second attempt (I can manually go there in my browser…there are no service blocks either)?
(Note I ultimately want to add additionalcitations to citations and return that combined list. What you see here doesn't take that into account as I am trying to debug).
import * as https from 'https'; // Importing the 'https' package
export const handler = async (event) => {
const serpapiKey = process.env.serpapikey; // Get the SerpAPI key from environment variables
const url = event.article_cited_by_serapi_link + '&api_key=' + serpapiKey; // Create the URL to make the SerpAPI request with the article_cited_by_serapi_link and API key
const response = await getPage(url); // Call the getPage function with the URL and wait for a response
const {paginationUrls,citations} = await getPaginationUrls(response, url); // Call the getPaginationUrls function with the response and URL to get an array of pagination URLs
//const additionalCitations = await getCitationsFromPaginationUrls(paginationUrls);
//return paginationUrls; // Return the pagination URLs array
return {citations,paginationUrls};
}
const getPage = async (url) => {
return new Promise((resolve, reject) => {
https.get(url, (res) => { // Make an HTTPS GET request to the specified URL using the 'https' package
let data = '';
res.on('data', (chunk) => {
data += chunk; // As data comes in, add it to the data variable
});
res.on('end', () => {
resolve(JSON.parse(data)); // When the response ends, parse the data and resolve the promise with the result
});
}).on('error', (err) => {
reject(err); // If there is an error, reject the promise with the error
});
});
};
//get the list of pagination URLs to later loop through
const getPaginationUrls = async (response, url) => {
let paginationUrls = []; // Create an empty array to hold the pagination URLs
let citations = [];
if (response.serpapi_pagination) { // If the response includes pagination information
const otherPages = response.serpapi_pagination.other_pages; // Get the 'other_pages' object from the pagination information
const numPages = Math.ceil(response.search_information.total_results / 10); // Calculate the number of pages based on the total number of results and the fact that each page shows 10 results
for (let i = 1; i <= numPages; i++) {
paginationUrls.push(`${url}&start=${10*(i-1)}`); // Add a URL to the paginationUrls array for each page, with the appropriate start parameter
}
}
// Extract citations from organic results
if (response.organic_results) {
citations = extractCitations(response);
}
return {paginationUrls,citations}; // Return the paginationUrls array
};