[tex]5x+2y=\frac{9x+7y}{2}\Leftrightarrow 10x+4y=9x+7y \Leftrightarrow x=3y[/tex]
[tex](y+2)^2=(x-1)(x-3y) \Leftrightarrow y^2+4y+4=x^2-3xy-x+3y \Leftrightarrow y^2+4y+4=9y^2-9y^2-3y+3y \Leftrightarrow y^2+4y+4=0[/tex]
=> [tex]y=-2[/tex]
=> [tex]x=-6[/tex]
=> [tex]x^2+y^2=4+36=40[/tex]