Telegram message import

The first script automatically retrieves and stores new Telegram messages using a specified API key, converting URLs in the messages to Markdown format, and then logs and updates each operation in a Fibery database.

The second script updates the relationship between Telegram message entities and group entities in a Fibery database, by matching and linking message entities to the correct group entity based on the group name, logging the result of each operation.

chrome_kMFlpJ2uKn

Script 1 (in Bot database)

// Configuration constants
const http = context.getService('http');
const fibery = context.getService('fibery');
const messageDB = 'Telegram/Message'; // Database for storing Telegram messages
const apiKeyDB = 'Yuri/API Key'; // Database for API Key storage
const apiKeyFieldName = 'API Key'; // Field name for the API Key in Bot entity
const botDB = 'Telegram/Bot'; // Database type for Bot entities
const lastUpdateIdFieldName = 'LastUpdateId'; // Field name for storing LastUpdateId in the API Key entity
const metaFieldName = 'Meta'; // Field name for the Meta rich text field

// Function to replace URLs with Markdown links
function replaceUrlsWithMarkdown(text) {
    const urlRegex = /(\bhttps?:\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])/ig;
    return text.replace(urlRegex, (url) => `[${url}](${url})`);
}

// Function to initialize or clear the Meta field log
async function clearAndInitMetaLog(botId) {
    const initialContent = `${new Date().toISOString()}: Script execution started.`;
    const entity = await fibery.getEntityById(botDB, botId, [metaFieldName]);
    await fibery.setDocumentContent(entity[metaFieldName]['Secret'], initialContent, 'md');
}

// Function to append to the Meta log
async function appendToMetaLog(botId, newContent) {
    const entity = await fibery.getEntityById(botDB, botId, [metaFieldName]);
    const currentMetaContent = entity[metaFieldName] ? await fibery.getDocumentContent(entity[metaFieldName]['Secret'], 'md') : '';
    const updatedContent = `${currentMetaContent}\n${new Date().toISOString()}: ${newContent}`;
    await fibery.setDocumentContent(entity[metaFieldName]['Secret'], updatedContent, 'md');
}

// Main function to fetch and save Telegram messages
async function fetchAndSaveTelegramMessages() {
    if (!args || !args.currentEntities || args.currentEntities.length === 0) {
        console.error("No Bot entity provided in args.currentEntities");
        return;
    }
    const currentBot = args.currentEntities[0];
    await clearAndInitMetaLog(currentBot.id);

    const apiKeyEntity = await fibery.getEntityById(apiKeyDB, currentBot[apiKeyFieldName].Id, ['Key', lastUpdateIdFieldName]);
    const telegramApiKey = apiKeyEntity.Key;
    const lastUpdateId = apiKeyEntity[lastUpdateIdFieldName] || "0";
    const displayApiKey = telegramApiKey ? `...${telegramApiKey.slice(-4)}` : "No API Key Found";

    await appendToMetaLog(currentBot.id, `Fetching messages using Telegram API Key: ${displayApiKey}`);
    const response = await http.getAsync(`https://api.telegram.org/bot${telegramApiKey}/getUpdates?offset=${parseInt(lastUpdateId) + 1}&limit=100`);
    const messages = JSON.parse(response).result;

    if (messages.length === 0) {
        await appendToMetaLog(currentBot.id, "No new messages found.");
        return;
    }

    for (const update of messages) {
        const message = update.message || update.edited_message;
        if (message && message.text) {
            let messageText = replaceUrlsWithMarkdown(message.text);
            const entityData = {
                'Name': 'Telegram Message',
                'Sender': message.from.first_name || message.from.username || "Unknown Sender",
                'Date Sent': new Date(message.date * 1000).toISOString(),
                'Group Name': message.chat.title || 'Private Chat',
                'Bot': currentBot.id  // Linking the message to the current Bot entity
            };
            const createdEntity = await fibery.createEntity(messageDB, entityData);

            // Fetch and log the public ID of the created message entity
            if (createdEntity && createdEntity['Public Id']) {
                await fibery.setDocumentContent(createdEntity['Description'].Secret, messageText, 'md');
                await appendToMetaLog(currentBot.id, `Message processed and linked: ${createdEntity.id} (Public ID: ${createdEntity['Public Id']})`);
            }

            if (update.update_id > parseInt(lastUpdateId)) {
                await fibery.updateEntity(apiKeyDB, apiKeyEntity.Id, { [lastUpdateIdFieldName]: update.update_id.toString() });
                await appendToMetaLog(currentBot.id, `Last update ID updated to: ${update.update_id}`);
            }
        }
    }
}

await fetchAndSaveTelegramMessages();

Script 1 explanation

  • Implement a function to convert plain URLs in a text to Markdown formatted links.
  • Create functions to clear and initialize a log in the Meta field, and to append new content to this log.
  • Define the main function to fetch and save Telegram messages:
  • Verify the presence of a Bot entity to proceed with operations.
  • Retrieve the API key and the last processed update ID from the Fibery database.
  • Log the initiation of message fetching using the retrieved API key.
  • Fetch new messages from Telegram using the API key and update offset.
  • For each received message, convert URLs to Markdown, create a new entity for the message in Fibery, and log the creation.
  • Update the last processed update ID in the database after processing each message.
  • Execute the main function to start the message fetching and saving process.

Script 2 (in Message database)

// Constants for database names and fields
const MESSAGE_DB = 'Telegram/Message';
const TGROUP_DB = 'Telegram/TGroup';
const GROUP_NAME_FIELD = 'Group Name'; // Field name in the message entity
const TGROUP_NAME_FIELD = 'Name'; // Field name in the TGroup entity
const BOT_TGROUPS_FIELD = 'Bot TGroups'; // Path to TGroup entities via Bot

const fibery = context.getService('fibery');

async function updateTGroupRelation() {
    try {
        const currentMessage = args.currentEntities[0];

        // Fetch 'Group Name' and related TGroups via 'Bot Tgroups'
        const messageEntity = await fibery.getEntityById(MESSAGE_DB, currentMessage.id, [GROUP_NAME_FIELD, BOT_TGROUPS_FIELD]);

        const groupName = messageEntity[GROUP_NAME_FIELD];
        const relatedTGroups = messageEntity[BOT_TGROUPS_FIELD];

        // Find a matching TGroup based on the 'Name' field
        const matchingTGroup = relatedTGroups.find(tGroup => tGroup[TGROUP_NAME_FIELD] === groupName);

        if (matchingTGroup) {
            // Update the 'TGroup' relation in the Message entity
            await fibery.updateEntity(MESSAGE_DB, currentMessage.id, {
                'TGroup': matchingTGroup.id
            });
            console.log(`TGroup updated to ${matchingTGroup.id} for Message with ID ${currentMessage.id}`);
        } else {
            console.log('No matching TGroup found for the provided Group Name:', groupName);
        }
    } catch (error) {
        console.error('Error in updating TGroup relation:', error);
    }
}

await updateTGroupRelation();

Script 2 explanation

Executes a function to update relationships between Telegram message entities and group entities:

  • Retrieves the current message entity along with its group name and related Telegram group entities.
  • Searches for a Telegram group entity that matches the group name of the current message.
  • Updates the message entity with the ID of the matching Telegram group entity if a match is found, or logs a message if no match is found.
4 Likes

What tool did you use for your hand-written style diagram? :slight_smile:

Thanks for the link!

1 Like