Merge pull request #615 from kolplattformen/feat/queue-fetcher

feat: 🎸 Fix wrong child response by queuing calls together

apps/api-test-app/README.md

@@ -0,0 +1,5 @@
+# To run the app use the following command line.
+# The arg is your personal id number for bankId identification
+# Observe the trailing comma! (it must be there, nx thing)
+
+nx serve api-test-app --args=19XXXXXXXX, 

apps/skolplattformen-app/components/image.component.tsx

@@ -42,20 +42,23 @@ export const Image = ({
       if (!url) return
       const newHeaders = await api.getSessionHeaders(url)
 
+      /*
       console.log('[IMAGE] Getting image dimensions with headers', {
         debugImageName,
         newHeaders,
       })
-
+*/
       ImageBase.getSizeWithHeaders(
         url,
         newHeaders,
         (w, h) => {
+          /*
           console.log('[IMAGE] Received image dimensions', {
             debugImageName,
             w,
             h,
           })
+          */
           setDimensions({ width: w, height: h })
           setHeaders(newHeaders)
         },

libs/api-skolplattformen/lib/__tests__/queueFetcher.test.ts

@@ -0,0 +1,55 @@
+import QueueFetcher from '../queue/queueFetcher'
+
+let sut : QueueFetcher
+beforeEach(() => {
+  jest.useFakeTimers('legacy')
+  sut = new QueueFetcher(async () => '')
+})
+
+test('creates queues for each id', () => {
+  sut.fetch(async () => '', 'one')
+  sut.fetch(async () => '', 'two')
+  sut.fetch(async () => '', 'three')
+
+  expect(sut.Queues).toHaveLength(3)
+})
+
+test('add same id to same queue', () => {
+  sut.fetch(async () => '', 'one')
+  sut.fetch(async () => '', 'one')
+  sut.fetch(async () => '', 'one')
+
+  expect(sut.Queues).toHaveLength(1)
+  expect(sut.Queues[0].id).toEqual('one')
+})
+
+test('can run a task', async () => {
+  const func = async () => 'output'
+  const promise = sut.fetch(func, 'one')
+
+  jest.runOnlyPendingTimers()
+
+  const result = await promise
+
+  expect(result).toEqual('output')
+})
+
+test('can run many tasks', async () => {
+  const promise1 = sut.fetch(async () => 'one', 'one')
+  const promise2 = sut.fetch(async () => 'two', 'two')
+  const promise3 = sut.fetch(async () => 'three', 'three')
+
+  await sut.schedule()
+  await sut.schedule()
+  await sut.schedule()
+
+  const result = await Promise.all([promise1, promise2, promise3])
+
+  expect(result).toEqual(['one', 'two', 'three'])
+})
+
+test('sets up timer on fetch', () => {
+  sut.fetch(async () => 'one', 'one')
+
+  expect(setTimeout).toHaveBeenCalledTimes(1)
+})

libs/api-skolplattformen/lib/api.ts

@@ -32,6 +32,7 @@ import * as html from 'node-html-parser'
 import * as fake from './fakeData'
 import { checkStatus, DummyStatusChecker } from './loginStatusChecker'
 import * as parse from './parse/index'
+import queueFetcherWrapper from './queueFetcherWrapper'
 import * as routes from './routes'
 
 const fakeResponse = <T>(data: T): Promise<T> =>
@@ -250,7 +251,15 @@ export class ApiSkolplattformen extends EventEmitter implements Api {
     }
 
     const data = await response.json()
-    return parse.children(data)
+
+    const parsed = parse.children(data)
+    const useSpecialQueueModeForFSChildren = parsed.some((c) => (c.status || '').includes('FS'))
+
+    if(useSpecialQueueModeForFSChildren) {
+      this.fetch = queueFetcherWrapper(this.fetch, (childId) => this.selectChildById(childId))
+    }
+
+    return parsed
   }
 
   public async getCalendar(child: EtjanstChild): Promise<CalendarItem[]> {
@@ -258,7 +267,7 @@ export class ApiSkolplattformen extends EventEmitter implements Api {
 
     const url = routes.calendar(child.id)
     const session = this.getRequestInit()
-    const response = await this.fetch('calendar', url, session)
+    const response = await this.fetch('calendar', url, session, child.id)
     const data = await response.json()
     return parse.calendar(data)
   }
@@ -325,7 +334,7 @@ export class ApiSkolplattformen extends EventEmitter implements Api {
 
     const url = routes.news(child.id)
     const session = this.getRequestInit()
-    const response = await this.fetch('news', url, session)
+    const response = await this.fetch('news', url, session, child.id)
 
     this.CheckResponseForCorrectChildStatus(response, child)
 
@@ -358,7 +367,7 @@ export class ApiSkolplattformen extends EventEmitter implements Api {
     }
     const url = routes.newsDetails(child.id, item.id)
     const session = this.getRequestInit()
-    const response = await this.fetch(`news_${item.id}`, url, session)
+    const response = await this.fetch(`news_${item.id}`, url, session, child.id)
 
     this.CheckResponseForCorrectChildStatus(response, child)
 
@@ -373,7 +382,7 @@ export class ApiSkolplattformen extends EventEmitter implements Api {
     if (menuService === 'rss') {
       const url = routes.menuRss(child.id)
       const session = this.getRequestInit()
-      const response = await this.fetch('menu-rss', url, session)
+      const response = await this.fetch('menu-rss', url, session, child.id)
 
       this.CheckResponseForCorrectChildStatus(response, child)
 
@@ -383,7 +392,7 @@ export class ApiSkolplattformen extends EventEmitter implements Api {
 
     const url = routes.menuList(child.id)
     const session = this.getRequestInit()
-    const response = await this.fetch('menu-list', url, session)
+    const response = await this.fetch('menu-list', url, session, child.id)
 
     this.CheckResponseForCorrectChildStatus(response, child)
 
@@ -394,7 +403,7 @@ export class ApiSkolplattformen extends EventEmitter implements Api {
   private async getMenuChoice(child: EtjanstChild): Promise<string> {
     const url = routes.menuChoice(child.id)
     const session = this.getRequestInit()
-    const response = await this.fetch('menu-choice', url, session)
+    const response = await this.fetch('menu-choice', url, session, child.id)
 
     this.CheckResponseForCorrectChildStatus(response, child)
 
@@ -559,7 +568,32 @@ export class ApiSkolplattformen extends EventEmitter implements Api {
     return parse.timetable(json, year, week, lang)
   }
 
+  public async selectChild(child : EtjanstChild): Promise<EtjanstChild> {
+    const response = await this.selectChildById(child.id)
 
+    const data = await response.json()
+    return parse.child(parse.etjanst(data))
+  }
+
+  private async selectChildById(childId: string) {
+    const requestInit = this.getRequestInit({
+      method: 'POST',
+      headers: {
+        host: 'etjanst.stockholm.se',
+        accept: 'application/json, text/plain, */*',
+        'accept-Encoding': 'gzip, deflate',
+        'content-Type': 'application/json;charset=UTF-8',
+        origin: 'https://etjanst.stockholm.se',
+        referer: 'https://etjanst.stockholm.se/vardnadshavare/inloggad2/hem',
+      },
+      body: JSON.stringify({
+        id: childId,
+      }),
+    })
+
+    const response = await this.fetch('selectChild', routes.selectChild, requestInit)
+    return response
+  }
 
   public async logout() {
     this.isFake = false

libs/api-skolplattformen/lib/fakeData/schoolContacts.ts

@@ -10,38 +10,38 @@ const schoolContactData = new Map<string, SchoolContact[]>([
     child1.id, [
       {
         title: "Expedition",
-        name: null,
+        name: undefined,
         phone: "508 000 00",
         email: "",
         schoolName: "Vallaskolan",
-        className: null,
+        className: '',
       },
       {
         title: "Rektor",
         name: "Alvar Sträng",
         phone: "08-50800001",
         email: "alvar.strang@edu.stockholm.se",
-        schoolName: null,
-        className: null,
+        schoolName: '',
+        className: '',
       }
   ]],
   [
     child2.id, [
     {
       title: "Expedition",
-      name: null,
+      name: undefined,
       phone: "508 000 00",
       email: "",
       schoolName: "Vallaskolan",
-      className: null,
+      className: '',
     },
     {
       title: "Rektor",
       name: "Alvar Sträng",
       phone: "08-50800001",
       email: "alvar.strang@edu.stockholm.se",
-      schoolName: null,
-      className: null,
+      schoolName: '',
+      className: '',
     }
 ]]
 ])

libs/api-skolplattformen/lib/fakeData/teachers.ts

@@ -6,15 +6,15 @@ export const teachers = (child: Child): Teacher[] => teacherData.get(child.id) ?
 const [child1,child2] = children()
 
 const teacherData = new Map<string, Teacher[]>([
-  [  
+  [
     child1.id, [
       {
         id: 15662220,
         firstname: "Cecilia",
-        sisId: null,
+        sisId: '',
         lastname: "Test",
         email: "cecilia.test@edu.stockholm.se",
-        phoneWork: null,
+        phoneWork: undefined,
         active: true,
         status: " S",
         timeTableAbbreviation: 'CTE',
@@ -23,7 +23,7 @@ const teacherData = new Map<string, Teacher[]>([
         id: 15662221,
         firstname: "Anna",
         lastname: "Test",
-        sisId: null,
+        sisId: '',
         email: "anna.test@edu.stockholm.se",
         phoneWork: '08000000',
         active: true,
@@ -34,8 +34,8 @@ const teacherData = new Map<string, Teacher[]>([
         id: 15662221,
         firstname: "Greta",
         lastname: "Test",
-        sisId: null,
-        email: null,
+        sisId: '',
+        email: undefined,
         phoneWork: '08000001',
         active: true,
         status: " F",
@@ -47,10 +47,10 @@ const teacherData = new Map<string, Teacher[]>([
     {
       id: 15662220,
       firstname: "Cecilia",
-      sisId: null,
+      sisId: '',
       lastname: "Test",
       email: "cecilia.test@edu.stockholm.se",
-      phoneWork: null,
+      phoneWork: undefined,
       active: true,
       status: " S",
       timeTableAbbreviation: 'CTE',
@@ -59,7 +59,7 @@ const teacherData = new Map<string, Teacher[]>([
       id: 15662221,
       firstname: "Anna",
       lastname: "Test",
-      sisId: null,
+      sisId: '',
       email: "anna.test@edu.stockholm.se",
       phoneWork: '08000000',
       active: true,
@@ -70,8 +70,8 @@ const teacherData = new Map<string, Teacher[]>([
       id: 15662221,
       firstname: "Greta",
       lastname: "Test",
-      sisId: null,
-      email: null,
+      sisId: '',
+      email: undefined,
       phoneWork: '08000001',
       active: true,
       status: " F",

libs/api-skolplattformen/lib/queue/autoQueue.ts

@@ -0,0 +1,95 @@
+import { Queue } from './queue'
+import { QueueStatus } from './queueStatus'
+
+export default class AutoQueue extends Queue {
+  private runningTasks: number
+
+  private maxConcurrentTasks: number
+
+  private isPaused: boolean
+
+  private queueStatus: QueueStatus
+
+  constructor(maxConcurrentTasks = 1) {
+    super()
+    this.runningTasks = 0
+    this.maxConcurrentTasks = maxConcurrentTasks
+    this.isPaused = false
+    this.queueStatus = new QueueStatus()
+  }
+
+  public enqueue<T>(action: () => Promise<T>, autoDequeue = true): Promise<T> {
+    return new Promise((resolve, reject) => {
+      super.enqueue({ action, resolve, reject })
+
+      if (autoDequeue) {
+        this.dequeue()
+      }
+    })
+  }
+
+  public async dequeue() {
+    if (this.runningTasks >= this.maxConcurrentTasks) { return false }
+
+    if (this.isPaused) { return false }
+
+    const item = super.dequeue()
+
+    if (!item) { return false }
+
+    try {
+      this.runningTasks += 1
+
+      const payload = await item.action(this)
+
+      this.decreaseRunningTasks()
+      item.resolve(payload)
+    } catch (e) {
+      this.decreaseRunningTasks()
+      item.reject(e)
+    } finally {
+      this.dequeue()
+    }
+
+    return true
+  }
+
+  public pause() {
+    this.isPaused = true
+  }
+
+  public async start() {
+    this.isPaused = false
+    // eslint-disable-next-line no-await-in-loop
+    while (await this.dequeue()) {
+      // do nothing
+    }
+  }
+
+  public get runningTaskCount() { return this.runningTasks }
+
+  public getQueueStatus() {
+    return this.queueStatus
+  }
+
+  public getQueueInfo() {
+    return {
+      itemsInQueue: this.size,
+      runningTasks: this.runningTasks,
+      isPaused: this.isPaused,
+    }
+  }
+
+  private decreaseRunningTasks() {
+    this.runningTasks -= 1
+
+    if (this.runningTasks <= 0) {
+      this.runningTasks = 0
+      this.queueStatus.emitIdleQueue()
+    }
+
+    if (this.size === 0) {
+      this.queueStatus.emitEmptyQueue()
+    }
+  }
+}

libs/api-skolplattformen/lib/queue/queue.ts

@@ -0,0 +1,11 @@
+export class Queue {
+  private items: any[]
+
+  constructor() { this.items = [] }
+
+  enqueue(item : any) { this.items.push(item) }
+
+  dequeue() { return this.items.shift() }
+
+  get size() { return this.items.length }
+}

libs/api-skolplattformen/lib/queue/queueFetcher.ts

@@ -0,0 +1,175 @@
+import AutoQueue from './autoQueue'
+import RoundRobinArray from './roundRobinArray'
+
+export interface QueueEntry {
+  id : string
+  queue : AutoQueue
+}
+
+function delay(time : any) {
+  return new Promise(resolve => setTimeout(resolve, time))
+}
+
+/**
+ * Put requests in queues where each childId gets its own queue
+ * The class takes care of calling the provided changeChildFunc
+ * before running the queue.
+ * Why? The external api uses state where the child must be selected
+ * before any calls to News etc can be done.
+ *
+*/
+export default class QueueFetcher {
+  private queues: RoundRobinArray<QueueEntry>
+
+  private currentRunningQueue : QueueEntry | undefined
+
+  private changeChildFunc : (childId : string) => Promise<any>
+
+  private lastChildId  = ''
+
+  private scheduleTimeout: any
+
+  /**
+   * Set to true to console.log verbose information
+   * For debugging mostly
+   */
+  verboseDebug = false
+
+  /**
+   * Creates a new QueueFetcher
+   * @param changeChildFunc function that is called to change the current
+   * selected child on the server
+   */
+  constructor(changeChildFunc : (childId : string) => Promise<any>) {
+    this.changeChildFunc = changeChildFunc
+    this.queues = new RoundRobinArray(new Array<QueueEntry>())
+  }
+
+  /**
+   * Queues a fetch - it will be executed together with other calls that
+   * has the same id
+   * @param func function that creates the request to be done. Must be a function
+   * because a Promise is always created in the running state
+   * @param id the id (e.g. childId) that is used to group calls together
+   * @returns a Promise that resolves when the Promise created by the func is resolved
+   * (i.e. is dequeued and executed)
+   */
+  public async fetch<T>(func : () => Promise<T>, id : string) : Promise<T> {
+    if (!this.queues.array.some((e) => e.id === id)) {
+      const newQueue = new AutoQueue(10)
+      this.queues.add({ id, queue: newQueue })
+    }
+
+    const queueEntry = this.queues.array.find((e) => e.id === id)
+    if (queueEntry === undefined) {
+      throw new Error(`No queue found for id: ${id}`)
+    }
+    const promise = queueEntry.queue.enqueue(func, false)
+
+    if (this.scheduleTimeout === undefined || this.scheduleTimeout === null) {
+      this.scheduleTimeout = setTimeout(async () => this.schedule(), 0)
+    }
+
+    return promise
+  }
+
+  public get Queues() { return this.queues.array }
+
+  /**
+   * Method to schedule next queue
+   * Public because we need it from unit-tests
+   */
+  async schedule() {
+    // Debug print info for all queues
+    this.queues.array.forEach(({ id: childId, queue }) => this.debug(
+      'Schedule: ',
+      childId, '=>', queue.getQueueInfo(),
+    ))
+
+    if (this.queues.size === 0) {
+      this.debug('No queues created yet')
+      return
+    }
+
+    if (this.currentRunningQueue === undefined || this.queues.size === 1) {
+      this.debug('First run schedule or only one queue')
+      const firstQueue = this.queues.first
+      await this.runNext(firstQueue)
+      return
+    }
+
+    const nextToRun = this.findNextQueueToRun()
+
+    if (nextToRun === undefined) {
+      this.debug('Nothing to do right now')
+      this.scheduleTimeout = null
+      return
+    }
+
+    if (nextToRun.id === this.currentRunningQueue.id) {
+      this.debug('Same queue as before was scheduled')
+      this.runNext(nextToRun)
+      return
+    }
+
+    const { id: queueToPauseId, queue: queueToPause } = this.currentRunningQueue
+    this.debug('Queue to pause', queueToPauseId, queueToPause.getQueueInfo())
+
+    queueToPause.pause()
+
+    if (queueToPause.runningTaskCount === 0) {
+      await this.runNext(nextToRun)
+      return
+    }
+
+    this.debug('Queue is not idle, waiting for it ...')
+
+    queueToPause.getQueueStatus().once('IDLE', async () => {
+      this.debug('Got IDLE from queue')
+      await this.runNext(nextToRun)
+    })
+  }
+
+  private async runNext(queueToRun : QueueEntry) {
+    const { id: childId, queue } = queueToRun
+    this.debug('About to run', childId, queue.getQueueInfo())
+
+
+    if (this.lastChildId === childId) {
+      this.debug('Child already selected, skipping select call')
+    } else {
+      this.debug('Initiating change child')
+      await this.changeChildFunc(childId)
+      this.lastChildId = childId
+      this.debug('Change child done')
+    }
+
+    this.currentRunningQueue = queueToRun
+
+    this.setupTimerForSchedule()
+    await queue.start()
+  }
+
+  private setupTimerForSchedule() {
+    this.scheduleTimeout = setTimeout(async () => this.schedule(), 3000)
+  }
+
+  private findNextQueueToRun() : QueueEntry | undefined {
+    // Iterate all queues and look for next queue with work to do
+    for (let i = 0; i < this.queues.size; i += 1) {
+      const { id: childId, queue } = this.queues.next()
+
+      // If queue has items to execute, return it
+      if (queue.size > 0 || queue.runningTaskCount > 0) return { id: childId, queue }
+    }
+
+    // Nothing more to do
+    return undefined
+  }
+
+  private debug(message : any, ...args : any[]) {
+    if (this.verboseDebug) {
+      console.debug(message, ...args)
+    }
+  }
+}

libs/api-skolplattformen/lib/queue/queueStatus.ts

@@ -0,0 +1,11 @@
+import { EventEmitter } from 'events'
+
+export class QueueStatus extends EventEmitter {
+  public emitEmptyQueue() {
+    this.emit('EMPTY')
+  }
+
+  public emitIdleQueue() {
+    this.emit('IDLE')
+  }
+}

libs/api-skolplattformen/lib/queue/roundRobinArray.ts

@@ -0,0 +1,30 @@
+export default class RoundRobinArray<T> {
+  index: any
+
+  array: T[]
+
+  constructor(array : Array<T>, index?: number | undefined) {
+    this.index = index || 0
+
+    if (array === undefined || array === null) {
+      this.array = new Array<T>()
+    } else if (!Array.isArray(array)) {
+      throw new Error('Expecting argument to RoundRound to be an Array')
+    }
+
+    this.array = array
+  }
+
+  next() {
+    this.index = (this.index + 1) % this.array.length
+    return this.array[this.index]
+  }
+
+  add(item : T) {
+    this.array.push(item)
+  }
+
+  get first() { return this.array[0] }
+
+  get size() { return this.array.length }
+}

libs/api-skolplattformen/lib/queueFetcherWrapper.ts

@@ -0,0 +1,19 @@
+import QueueFetcher from './queue/queueFetcher'
+import {Fetcher, RequestInit, Response } from '@skolplattformen/api'
+
+
+export default function queueFetcherWrapper(fetch: Fetcher,
+  changeChildFunc: ((childId: string) => Promise<Response>)) : Fetcher {
+  const queue = new QueueFetcher(changeChildFunc)
+  queue.verboseDebug = false
+
+  return async (name: string, url: string, init: RequestInit = { headers: {} }, childId? : string)
+  : Promise<Response> => {
+    if (childId === undefined) {
+      return fetch(name, url, init)
+    }
+
+    const p = queue.fetch(() => fetch(name, url, init), childId)
+    return p
+  }
+}

libs/api-skolplattformen/lib/routes.ts

@@ -74,8 +74,8 @@ export const createItemConfig =
 
 // Skola24
 export const ssoRequestUrl = (targetSystem: string) =>
- `https://fnsservicesso1.stockholm.se/sso-ng/saml-2.0/authenticate?customer=https://login001.stockholm.se&targetsystem=${targetSystem}`  
-    
+ `https://fnsservicesso1.stockholm.se/sso-ng/saml-2.0/authenticate?customer=https://login001.stockholm.se&targetsystem=${targetSystem}`
+
 export const ssoResponseUrl = 'https://login001.stockholm.se/affwebservices/public/saml2sso'
 export const samlResponseUrl = 'https://fnsservicesso1.stockholm.se/sso-ng/saml-2.0/response'
 
@@ -83,4 +83,6 @@ export const timetables = 'https://fns.stockholm.se/ng/api/services/skola24/get/
 export const renderKey = 'https://fns.stockholm.se/ng/api/get/timetable/render/key'
 export const timetable = 'https://fns.stockholm.se/ng/api/render/timetable'
 
-export const topologyConfigUrl = 'https://fantomenkrypto.vercel.app/api/getConfig'
+export const topologyConfigUrl = 'https://fantomenkrypto.vercel.app/api/getConfig'
+
+export const selectChild = 'https://etjanst.stockholm.se/vardnadshavare/inloggad2/SelectChild'

libs/api/lib/fetcher.ts

@@ -17,7 +17,7 @@ export interface FetcherOptions {
 }
 
 export interface Fetcher {
-  (name: string, url: string, init?: RequestInit): Promise<Response>
+  (name: string, url: string, init?: RequestInit, childId?: string): Promise<Response>
 }
 
 export interface Recorder {